You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2021/02/07 13:48:24 UTC
[kylin] branch kylin-on-parquet-v2 updated: KYLIN-4898 Automated
test
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this push:
new 0f94e81 KYLIN-4898 Automated test
0f94e81 is described below
commit 0f94e8165427eeb0cbae516f9516d4742f7a1205
Author: helenzeng0503 <li...@163.com>
AuthorDate: Fri Jan 29 16:38:09 2021 +0800
KYLIN-4898 Automated test
---
.../data/auto_config/auto_config_no_distinct.json | 420 ++++++++++++++++++++
.../auto_config/auto_config_override_conf.json | 423 +++++++++++++++++++++
.../generic_desc_data/generic_desc_data_3x.json | 0
.../generic_desc_data/generic_desc_data_4x.json | 187 +++++++++
.../data/happy_path/happy_path.sql | 43 +++
.../env/default/python.properties | 2 +-
.../features/specs/auto_config/auto_config.spec | 13 +
.../features/specs/happy_path/happy_path.spec | 5 +
.../features/specs/project_model/model.spec | 7 +
.../features/specs/project_model/project.spec | 11 +
.../features/step_impl/auto_config/auto_config.py | 196 ++++++++++
.../features/step_impl/before_suite.py | 8 +
.../features/step_impl/happy_path/happy_path.py | 68 ++++
.../features/step_impl/project_model/model.py | 28 ++
.../features/step_impl/project_model/project.py | 46 +++
.../{kylin_instance.yml => kylin_host.yml} | 9 +-
.../kylin_instances/kylin_instance.yml | 2 +-
.../CI/kylin-system-testing/kylin_utils/equals.py | 2 +-
build/CI/kylin-system-testing/kylin_utils/shell.py | 5 -
build/CI/kylin-system-testing/kylin_utils/util.py | 11 +-
20 files changed, 1471 insertions(+), 15 deletions(-)
diff --git a/build/CI/kylin-system-testing/data/auto_config/auto_config_no_distinct.json b/build/CI/kylin-system-testing/data/auto_config/auto_config_no_distinct.json
new file mode 100644
index 0000000..7ef7f0c
--- /dev/null
+++ b/build/CI/kylin-system-testing/data/auto_config/auto_config_no_distinct.json
@@ -0,0 +1,420 @@
+{
+ "cube_desc_data":
+ {
+ "uuid": "788f16f4-94b2-f7f4-0a62-5dc8a02ceee8",
+ "last_modified": 0,
+ "version": "4.0.0.0",
+ "name": "kylin_sales_cube_no_distinct",
+ "is_draft": false,
+ "model_name": "kylin_sales_model",
+ "description": "",
+ "null_string": null,
+ "dimensions": [
+ {
+ "name": "TRANS_ID",
+ "table": "KYLIN_SALES",
+ "column": "TRANS_ID",
+ "derived": null
+ },
+ {
+ "name": "YEAR_BEG_DT",
+ "table": "KYLIN_CAL_DT",
+ "column": null,
+ "derived": [
+ "YEAR_BEG_DT"
+ ]
+ },
+ {
+ "name": "MONTH_BEG_DT",
+ "table": "KYLIN_CAL_DT",
+ "column": null,
+ "derived": [
+ "MONTH_BEG_DT"
+ ]
+ },
+ {
+ "name": "WEEK_BEG_DT",
+ "table": "KYLIN_CAL_DT",
+ "column": null,
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
+ },
+ {
+ "name": "USER_DEFINED_FIELD1",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": null,
+ "derived": [
+ "USER_DEFINED_FIELD1"
+ ]
+ },
+ {
+ "name": "USER_DEFINED_FIELD3",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": null,
+ "derived": [
+ "USER_DEFINED_FIELD3"
+ ]
+ },
+ {
+ "name": "META_CATEG_NAME",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEG_LVL2_NAME",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEG_LVL3_NAME",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
+ },
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "KYLIN_SALES",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+ },
+ {
+ "name": "SELLER_ID",
+ "table": "KYLIN_SALES",
+ "column": "SELLER_ID",
+ "derived": null
+ },
+ {
+ "name": "BUYER_ID",
+ "table": "KYLIN_SALES",
+ "column": "BUYER_ID",
+ "derived": null
+ },
+ {
+ "name": "ACCOUNT_BUYER_LEVEL",
+ "table": "BUYER_ACCOUNT",
+ "column": "ACCOUNT_BUYER_LEVEL",
+ "derived": null
+ },
+ {
+ "name": "ACCOUNT_SELLER_LEVEL",
+ "table": "SELLER_ACCOUNT",
+ "column": "ACCOUNT_SELLER_LEVEL",
+ "derived": null
+ },
+ {
+ "name": "BUYER_COUNTRY",
+ "table": "BUYER_ACCOUNT",
+ "column": "ACCOUNT_COUNTRY",
+ "derived": null
+ },
+ {
+ "name": "SELLER_COUNTRY",
+ "table": "SELLER_ACCOUNT",
+ "column": "ACCOUNT_COUNTRY",
+ "derived": null
+ },
+ {
+ "name": "BUYER_COUNTRY_NAME",
+ "table": "BUYER_COUNTRY",
+ "column": "NAME",
+ "derived": null
+ },
+ {
+ "name": "SELLER_COUNTRY_NAME",
+ "table": "SELLER_COUNTRY",
+ "column": "NAME",
+ "derived": null
+ },
+ {
+ "name": "OPS_USER_ID",
+ "table": "KYLIN_SALES",
+ "column": "OPS_USER_ID",
+ "derived": null
+ },
+ {
+ "name": "OPS_REGION",
+ "table": "KYLIN_SALES",
+ "column": "OPS_REGION",
+ "derived": null
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "KYLIN_SALES.PRICE"
+ },
+ "returntype": "decimal(19,4)"
+ }
+ },
+ {
+ "name": "BUYER_LEVEL_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL"
+ },
+ "returntype": "bigint"
+ }
+ },
+ {
+ "name": "SELLER_LEVEL_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"
+ },
+ "returntype": "bigint"
+ }
+ },
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1"
+ },
+ "returntype": "bigint"
+ }
+ },
+ {
+ "name": "TOP_SELLER",
+ "function": {
+ "expression": "TOP_N",
+ "parameter": {
+ "type": "column",
+ "value": "KYLIN_SALES.PRICE",
+ "next_parameter": {
+ "type": "column",
+ "value": "KYLIN_SALES.SELLER_ID"
+ }
+ },
+ "returntype": "topn(100,4)",
+ "configuration": {
+ "topn.encoding.KYLIN_SALES.SELLER_ID": "dict",
+ "topn.encoding_version.KYLIN_SALES.SELLER_ID": "1"
+ }
+ }
+ },
+ {
+ "name": "PRICE_PERCENTILE",
+ "function": {
+ "expression": "PERCENTILE_APPROX",
+ "parameter": {
+ "type": "column",
+ "value": "KYLIN_SALES.PRICE"
+ },
+ "returntype": "percentile(100)"
+ }
+ }
+ ],
+ "dictionaries": [],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "KYLIN_SALES.BUYER_ID",
+ "encoding": "integer:4",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.SELLER_ID",
+ "encoding": "integer:4",
+ "isShardBy": true
+ },
+ {
+ "column": "KYLIN_SALES.TRANS_ID",
+ "encoding": "integer:4",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.PART_DT",
+ "encoding": "date",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.LEAF_CATEG_ID",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "BUYER_COUNTRY.NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "SELLER_COUNTRY.NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.LSTG_FORMAT_NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.LSTG_SITE_ID",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.OPS_USER_ID",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.OPS_REGION",
+ "encoding": "dict",
+ "isShardBy": false
+ }
+ ]
+ },
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "F1",
+ "columns": [
+ {
+ "qualifier": "M",
+ "measure_refs": [
+ "GMV_SUM",
+ "BUYER_LEVEL_SUM",
+ "SELLER_LEVEL_SUM",
+ "TRANS_CNT"
+ ]
+ }
+ ]
+ },
+ {
+ "name": "F2",
+ "columns": [
+ {
+ "qualifier": "M",
+ "measure_refs": [
+ "TOP_SELLER",
+ "PRICE_PERCENTILE"
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "aggregation_groups": [
+ {
+ "includes": [
+ "KYLIN_SALES.PART_DT",
+ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "KYLIN_SALES.LEAF_CATEG_ID",
+ "KYLIN_SALES.LSTG_FORMAT_NAME",
+ "KYLIN_SALES.LSTG_SITE_ID",
+ "KYLIN_SALES.OPS_USER_ID",
+ "KYLIN_SALES.OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME",
+ "SELLER_COUNTRY.NAME"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "KYLIN_SALES.LEAF_CATEG_ID"
+ ]
+ ],
+ "mandatory_dims": [
+ "KYLIN_SALES.PART_DT"
+ ],
+ "joint_dims": [
+ [
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME"
+ ],
+ [
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME"
+ ],
+ [
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"
+ ],
+ [
+ "KYLIN_SALES.LSTG_FORMAT_NAME",
+ "KYLIN_SALES.LSTG_SITE_ID"
+ ],
+ [
+ "KYLIN_SALES.OPS_USER_ID",
+ "KYLIN_SALES.OPS_REGION"
+ ]
+ ]
+ }
+ }
+ ],
+ "signature": "tADjcemRs3OW+O5Qdhf0gQ==",
+ "notify_list": [],
+ "status_need_notify": [],
+ "partition_date_start": 1325376000000,
+ "partition_date_end": 3153600000000,
+ "auto_merge_time_ranges": [],
+ "volatile_range": 0,
+ "retention_range": 0,
+ "engine_type": 6,
+ "storage_type": 4,
+ "override_kylin_properties": {
+ "kylin.cube.aggrgroup.is-mandatory-only-valid": "true",
+ "kylin.engine.spark.rdd-partition-cut-mb": "500"
+ },
+ "cuboid_black_list": [],
+ "parent_forward": 3,
+ "mandatory_dimension_set_list": [],
+ "snapshot_table_desc_list": []
+ }
+}
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/data/auto_config/auto_config_override_conf.json b/build/CI/kylin-system-testing/data/auto_config/auto_config_override_conf.json
new file mode 100644
index 0000000..188c140
--- /dev/null
+++ b/build/CI/kylin-system-testing/data/auto_config/auto_config_override_conf.json
@@ -0,0 +1,423 @@
+{
+ "cube_desc_data":
+ {
+ "uuid": "788f16f4-94b2-f7f4-0a62-5dc8a02ceee8",
+ "last_modified": 0,
+ "version": "4.0.0.0",
+ "name": "kylin_sales_cube_override_conf",
+ "is_draft": false,
+ "model_name": "kylin_sales_model",
+ "description": "",
+ "null_string": null,
+ "dimensions": [
+ {
+ "name": "TRANS_ID",
+ "table": "KYLIN_SALES",
+ "column": "TRANS_ID",
+ "derived": null
+ },
+ {
+ "name": "YEAR_BEG_DT",
+ "table": "KYLIN_CAL_DT",
+ "column": null,
+ "derived": [
+ "YEAR_BEG_DT"
+ ]
+ },
+ {
+ "name": "MONTH_BEG_DT",
+ "table": "KYLIN_CAL_DT",
+ "column": null,
+ "derived": [
+ "MONTH_BEG_DT"
+ ]
+ },
+ {
+ "name": "WEEK_BEG_DT",
+ "table": "KYLIN_CAL_DT",
+ "column": null,
+ "derived": [
+ "WEEK_BEG_DT"
+ ]
+ },
+ {
+ "name": "USER_DEFINED_FIELD1",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": null,
+ "derived": [
+ "USER_DEFINED_FIELD1"
+ ]
+ },
+ {
+ "name": "USER_DEFINED_FIELD3",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": null,
+ "derived": [
+ "USER_DEFINED_FIELD3"
+ ]
+ },
+ {
+ "name": "META_CATEG_NAME",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": "META_CATEG_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEG_LVL2_NAME",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL2_NAME",
+ "derived": null
+ },
+ {
+ "name": "CATEG_LVL3_NAME",
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "column": "CATEG_LVL3_NAME",
+ "derived": null
+ },
+ {
+ "name": "LSTG_FORMAT_NAME",
+ "table": "KYLIN_SALES",
+ "column": "LSTG_FORMAT_NAME",
+ "derived": null
+ },
+ {
+ "name": "SELLER_ID",
+ "table": "KYLIN_SALES",
+ "column": "SELLER_ID",
+ "derived": null
+ },
+ {
+ "name": "BUYER_ID",
+ "table": "KYLIN_SALES",
+ "column": "BUYER_ID",
+ "derived": null
+ },
+ {
+ "name": "ACCOUNT_BUYER_LEVEL",
+ "table": "BUYER_ACCOUNT",
+ "column": "ACCOUNT_BUYER_LEVEL",
+ "derived": null
+ },
+ {
+ "name": "ACCOUNT_SELLER_LEVEL",
+ "table": "SELLER_ACCOUNT",
+ "column": "ACCOUNT_SELLER_LEVEL",
+ "derived": null
+ },
+ {
+ "name": "BUYER_COUNTRY",
+ "table": "BUYER_ACCOUNT",
+ "column": "ACCOUNT_COUNTRY",
+ "derived": null
+ },
+ {
+ "name": "SELLER_COUNTRY",
+ "table": "SELLER_ACCOUNT",
+ "column": "ACCOUNT_COUNTRY",
+ "derived": null
+ },
+ {
+ "name": "BUYER_COUNTRY_NAME",
+ "table": "BUYER_COUNTRY",
+ "column": "NAME",
+ "derived": null
+ },
+ {
+ "name": "SELLER_COUNTRY_NAME",
+ "table": "SELLER_COUNTRY",
+ "column": "NAME",
+ "derived": null
+ },
+ {
+ "name": "OPS_USER_ID",
+ "table": "KYLIN_SALES",
+ "column": "OPS_USER_ID",
+ "derived": null
+ },
+ {
+ "name": "OPS_REGION",
+ "table": "KYLIN_SALES",
+ "column": "OPS_REGION",
+ "derived": null
+ }
+ ],
+ "measures": [
+ {
+ "name": "GMV_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "KYLIN_SALES.PRICE"
+ },
+ "returntype": "decimal(19,4)"
+ }
+ },
+ {
+ "name": "BUYER_LEVEL_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL"
+ },
+ "returntype": "bigint"
+ }
+ },
+ {
+ "name": "SELLER_LEVEL_SUM",
+ "function": {
+ "expression": "SUM",
+ "parameter": {
+ "type": "column",
+ "value": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"
+ },
+ "returntype": "bigint"
+ }
+ },
+ {
+ "name": "TRANS_CNT",
+ "function": {
+ "expression": "COUNT",
+ "parameter": {
+ "type": "constant",
+ "value": "1"
+ },
+ "returntype": "bigint"
+ }
+ },
+ {
+ "name": "TOP_SELLER",
+ "function": {
+ "expression": "TOP_N",
+ "parameter": {
+ "type": "column",
+ "value": "KYLIN_SALES.PRICE",
+ "next_parameter": {
+ "type": "column",
+ "value": "KYLIN_SALES.SELLER_ID"
+ }
+ },
+ "returntype": "topn(100,4)",
+ "configuration": {
+ "topn.encoding.KYLIN_SALES.SELLER_ID": "dict",
+ "topn.encoding_version.KYLIN_SALES.SELLER_ID": "1"
+ }
+ }
+ },
+ {
+ "name": "PRICE_PERCENTILE",
+ "function": {
+ "expression": "PERCENTILE_APPROX",
+ "parameter": {
+ "type": "column",
+ "value": "KYLIN_SALES.PRICE"
+ },
+ "returntype": "percentile(100)"
+ }
+ }
+ ],
+ "dictionaries": [],
+ "rowkey": {
+ "rowkey_columns": [
+ {
+ "column": "KYLIN_SALES.BUYER_ID",
+ "encoding": "integer:4",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.SELLER_ID",
+ "encoding": "integer:4",
+ "isShardBy": true
+ },
+ {
+ "column": "KYLIN_SALES.TRANS_ID",
+ "encoding": "integer:4",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.PART_DT",
+ "encoding": "date",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.LEAF_CATEG_ID",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "BUYER_COUNTRY.NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "SELLER_COUNTRY.NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.LSTG_FORMAT_NAME",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.LSTG_SITE_ID",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.OPS_USER_ID",
+ "encoding": "dict",
+ "isShardBy": false
+ },
+ {
+ "column": "KYLIN_SALES.OPS_REGION",
+ "encoding": "dict",
+ "isShardBy": false
+ }
+ ]
+ },
+ "hbase_mapping": {
+ "column_family": [
+ {
+ "name": "F1",
+ "columns": [
+ {
+ "qualifier": "M",
+ "measure_refs": [
+ "GMV_SUM",
+ "BUYER_LEVEL_SUM",
+ "SELLER_LEVEL_SUM",
+ "TRANS_CNT"
+ ]
+ }
+ ]
+ },
+ {
+ "name": "F2",
+ "columns": [
+ {
+ "qualifier": "M",
+ "measure_refs": [
+ "TOP_SELLER",
+ "PRICE_PERCENTILE"
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "aggregation_groups": [
+ {
+ "includes": [
+ "KYLIN_SALES.PART_DT",
+ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "KYLIN_SALES.LEAF_CATEG_ID",
+ "KYLIN_SALES.LSTG_FORMAT_NAME",
+ "KYLIN_SALES.LSTG_SITE_ID",
+ "KYLIN_SALES.OPS_USER_ID",
+ "KYLIN_SALES.OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME",
+ "SELLER_COUNTRY.NAME"
+ ],
+ "select_rule": {
+ "hierarchy_dims": [
+ [
+ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "KYLIN_SALES.LEAF_CATEG_ID"
+ ]
+ ],
+ "mandatory_dims": [
+ "KYLIN_SALES.PART_DT"
+ ],
+ "joint_dims": [
+ [
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME"
+ ],
+ [
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME"
+ ],
+ [
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"
+ ],
+ [
+ "KYLIN_SALES.LSTG_FORMAT_NAME",
+ "KYLIN_SALES.LSTG_SITE_ID"
+ ],
+ [
+ "KYLIN_SALES.OPS_USER_ID",
+ "KYLIN_SALES.OPS_REGION"
+ ]
+ ]
+ }
+ }
+ ],
+ "signature": "tADjcemRs3OW+O5Qdhf0gQ==",
+ "notify_list": [],
+ "status_need_notify": [],
+ "partition_date_start": 1325376000000,
+ "partition_date_end": 3153600000000,
+ "auto_merge_time_ranges": [],
+ "volatile_range": 0,
+ "retention_range": 0,
+ "engine_type": 6,
+ "storage_type": 4,
+ "override_kylin_properties": {
+ "kylin.engine.spark-conf.spark.executor.memory": "2G",
+ "kylin.engine.spark-conf.spark.executor.cores": "2",
+ "kylin.engine.spark-conf.spark.executor.memoryOverhead": "256M",
+ "kylin.engine.spark-conf.spark.executor.instances": "3",
+ "kylin.engine.spark-conf.spark.sql.shuffle.partitions": "3"
+ },
+ "cuboid_black_list": [],
+ "parent_forward": 3,
+ "mandatory_dimension_set_list": [],
+ "snapshot_table_desc_list": []
+ }
+}
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_3x.json b/build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_3x.json
similarity index 100%
rename from build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_3x.json
rename to build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_3x.json
diff --git a/build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_4x.json b/build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_4x.json
similarity index 81%
rename from build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_4x.json
rename to build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_4x.json
index f322325..abecc0b 100644
--- a/build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_4x.json
+++ b/build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_4x.json
@@ -192,6 +192,193 @@
"capacity": "MEDIUM",
"projectName": "generic_test_project"
},
+ "snowflake_left_incre_model":
+ {
+ "uuid": "c465bc68-46fa-cc0b-254d-58383c01557a",
+ "last_modified": 0,
+ "version": "4.0.0.0",
+ "name": "snowflake_left_incre_model",
+ "owner": "ADMIN",
+ "is_draft": false,
+ "description": "Snowflakes on top of KylinSales.",
+ "fact_table": "DEFAULT.KYLIN_SALES",
+ "lookups": [
+ {
+ "table": "DEFAULT.KYLIN_CAL_DT",
+ "kind": "LOOKUP",
+ "alias": "KYLIN_CAL_DT",
+ "join": {
+ "type": "left",
+ "primary_key": [
+ "KYLIN_CAL_DT.CAL_DT"
+ ],
+ "foreign_key": [
+ "KYLIN_SALES.PART_DT"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "kind": "LOOKUP",
+ "alias": "KYLIN_CATEGORY_GROUPINGS",
+ "join": {
+ "type": "left",
+ "primary_key": [
+ "KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID",
+ "KYLIN_CATEGORY_GROUPINGS.SITE_ID"
+ ],
+ "foreign_key": [
+ "KYLIN_SALES.LEAF_CATEG_ID",
+ "KYLIN_SALES.LSTG_SITE_ID"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.KYLIN_ACCOUNT",
+ "kind": "LOOKUP",
+ "alias": "BUYER_ACCOUNT",
+ "join": {
+ "type": "left",
+ "primary_key": [
+ "BUYER_ACCOUNT.ACCOUNT_ID"
+ ],
+ "foreign_key": [
+ "KYLIN_SALES.BUYER_ID"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.KYLIN_ACCOUNT",
+ "kind": "LOOKUP",
+ "alias": "SELLER_ACCOUNT",
+ "join": {
+ "type": "left",
+ "primary_key": [
+ "SELLER_ACCOUNT.ACCOUNT_ID"
+ ],
+ "foreign_key": [
+ "KYLIN_SALES.SELLER_ID"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.KYLIN_COUNTRY",
+ "kind": "LOOKUP",
+ "alias": "BUYER_COUNTRY",
+ "join": {
+ "type": "left",
+ "primary_key": [
+ "BUYER_COUNTRY.COUNTRY"
+ ],
+ "foreign_key": [
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.KYLIN_COUNTRY",
+ "kind": "LOOKUP",
+ "alias": "SELLER_COUNTRY",
+ "join": {
+ "type": "left",
+ "primary_key": [
+ "SELLER_COUNTRY.COUNTRY"
+ ],
+ "foreign_key": [
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY"
+ ]
+ }
+ }
+ ],
+ "dimensions": [
+ {
+ "table": "KYLIN_SALES",
+ "columns": [
+ "TRANS_ID",
+ "SELLER_ID",
+ "BUYER_ID",
+ "PART_DT",
+ "LEAF_CATEG_ID",
+ "LSTG_FORMAT_NAME",
+ "LSTG_SITE_ID",
+ "OPS_USER_ID",
+ "OPS_REGION"
+ ]
+ },
+ {
+ "table": "KYLIN_CAL_DT",
+ "columns": [
+ "CAL_DT",
+ "WEEK_BEG_DT",
+ "MONTH_BEG_DT",
+ "YEAR_BEG_DT"
+ ]
+ },
+ {
+ "table": "KYLIN_CATEGORY_GROUPINGS",
+ "columns": [
+ "USER_DEFINED_FIELD1",
+ "USER_DEFINED_FIELD3",
+ "META_CATEG_NAME",
+ "CATEG_LVL2_NAME",
+ "CATEG_LVL3_NAME",
+ "LEAF_CATEG_ID",
+ "SITE_ID"
+ ]
+ },
+ {
+ "table": "BUYER_ACCOUNT",
+ "columns": [
+ "ACCOUNT_ID",
+ "ACCOUNT_BUYER_LEVEL",
+ "ACCOUNT_SELLER_LEVEL",
+ "ACCOUNT_COUNTRY",
+ "ACCOUNT_CONTACT"
+ ]
+ },
+ {
+ "table": "SELLER_ACCOUNT",
+ "columns": [
+ "ACCOUNT_ID",
+ "ACCOUNT_BUYER_LEVEL",
+ "ACCOUNT_SELLER_LEVEL",
+ "ACCOUNT_COUNTRY",
+ "ACCOUNT_CONTACT"
+ ]
+ },
+ {
+ "table": "BUYER_COUNTRY",
+ "columns": [
+ "COUNTRY",
+ "NAME"
+ ]
+ },
+ {
+ "table": "SELLER_COUNTRY",
+ "columns": [
+ "COUNTRY",
+ "NAME"
+ ]
+ }
+ ],
+ "metrics": [
+ "KYLIN_SALES.PRICE",
+ "KYLIN_SALES.ITEM_COUNT",
+ "KYLIN_SALES.ITEM_ID"
+ ],
+ "filter_condition": "KYLIN_SALES.PRICE > 25.0",
+ "partition_desc": {
+ "partition_date_column": "KYLIN_SALES.PART_DT",
+ "partition_time_column": null,
+ "partition_date_start": 1325376000000,
+ "partition_date_format": "yyyy-MM-dd",
+ "partition_time_format": "HH:mm:ss",
+ "partition_type": "APPEND",
+ "partition_condition_builder": "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
+ },
+ "capacity": "MEDIUM",
+ "projectName": "generic_test_project"
+ },
"cube_desc_data":
{
"uuid": "86da18cb-6eb9-bf43-5c69-aae88a7b5915",
diff --git a/build/CI/kylin-system-testing/data/happy_path/happy_path.sql b/build/CI/kylin-system-testing/data/happy_path/happy_path.sql
new file mode 100644
index 0000000..4fd469b
--- /dev/null
+++ b/build/CI/kylin-system-testing/data/happy_path/happy_path.sql
@@ -0,0 +1,43 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+KYLIN_SALES.PART_DT,
+KYLIN_CAL_DT.YEAR_BEG_DT,
+SUM(BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL) as BUYER_LEVEL_SUM,
+COUNT(DISTINCT KYLIN_SALES.SELLER_ID) as SELLER_CNT_HLL,
+SUM(KYLIN_SALES.PRICE) as GMV,
+PERCENTILE(KYLIN_SALES.PRICE, 0.5) as PRICE_PERCENTILE,
+COUNT(DISTINCT KYLIN_SALES.ITEM_ID) as ITEM_COUNT
+FROM KYLIN_SALES
+INNER JOIN KYLIN_CAL_DT as KYLIN_CAL_DT
+ON KYLIN_SALES.PART_DT = KYLIN_CAL_DT.CAL_DT
+INNER JOIN KYLIN_CATEGORY_GROUPINGS
+ON KYLIN_SALES.LEAF_CATEG_ID = KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND KYLIN_SALES.LSTG_SITE_ID = KYLIN_CATEGORY_GROUPINGS.SITE_ID
+INNER JOIN KYLIN_ACCOUNT as BUYER_ACCOUNT
+ON KYLIN_SALES.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+INNER JOIN KYLIN_ACCOUNT as SELLER_ACCOUNT
+ON KYLIN_SALES.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+INNER JOIN KYLIN_COUNTRY as BUYER_COUNTRY
+ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+INNER JOIN KYLIN_COUNTRY as SELLER_COUNTRY
+ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
+WHERE PART_DT >= '2012-12-25' and PART_DT < '2013-01-05' and SELLER_COUNTRY.COUNTRY in ('CN')
+GROUP BY KYLIN_SALES.PART_DT, KYLIN_CAL_DT.YEAR_BEG_DT
+ORDER BY KYLIN_SALES.PART_DT, KYLIN_CAL_DT.YEAR_BEG_DT;
+
diff --git a/build/CI/kylin-system-testing/env/default/python.properties b/build/CI/kylin-system-testing/env/default/python.properties
index 4dc60a9..1432d27 100644
--- a/build/CI/kylin-system-testing/env/default/python.properties
+++ b/build/CI/kylin-system-testing/env/default/python.properties
@@ -15,7 +15,7 @@
# limitations under the License.
#
-GAUGE_PYTHON_COMMAND = python3
+GAUGE_PYTHON_COMMAND = python
# Comma seperated list of dirs. path should be relative to project root.
STEP_IMPL_DIR = features/step_impl
diff --git a/build/CI/kylin-system-testing/features/specs/auto_config/auto_config.spec b/build/CI/kylin-system-testing/features/specs/auto_config/auto_config.spec
new file mode 100644
index 0000000..a47e57a
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/specs/auto_config/auto_config.spec
@@ -0,0 +1,13 @@
+# Automically setting spark configurations
+Tags: auto_config, 4.x
+
+## use default configuration
+* use default configuration on cube "kylin_sales_cube"
+## auto configuration with user-defined parameters
+* auto configuration with user-defined parameters on cube "kylin_sales_cube"
+## auto configuration with user-defined parameters
+* auto configuration with user-defined parameters on "kylin_sales_cube" and "kylin_sales_cube_no_distinct"
+## auto_config and override on cube level
+* auto_config and override on cube level on cube "kylin_sales_cube_override_conf"
+//## auto configuration off, override on cube level
+//* auto configuration off, override on cube level
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/features/specs/happy_path/happy_path.spec b/build/CI/kylin-system-testing/features/specs/happy_path/happy_path.spec
new file mode 100644
index 0000000..8526704
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/specs/happy_path/happy_path.spec
@@ -0,0 +1,5 @@
+# Automated test for Happy Path
+Tags: happy_path, 4.x
+
+## automated happy path
+* automated happy path, check query using "meta_data/happy_path/happy_path.sql"
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/features/specs/project_model/model.spec b/build/CI/kylin-system-testing/features/specs/project_model/model.spec
new file mode 100644
index 0000000..507230c
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/specs/project_model/model.spec
@@ -0,0 +1,7 @@
+# model management
+Tags: model, 4.x
+
+## model clone
+* in project "generic_test_project", clone model "generic_test_model" and name it "model_name_clone"
+## model clone duplicated
+* again, in project "generic_test_project", clone model "generic_test_model" and name it "model_name_clone"
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/features/specs/project_model/project.spec b/build/CI/kylin-system-testing/features/specs/project_model/project.spec
new file mode 100644
index 0000000..e094a94
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/specs/project_model/project.spec
@@ -0,0 +1,11 @@
+# project management
+Tags: project, 4.x
+
+## create duplicate project
+* create a project "test_project_description" with description "this is a test project" and check that duplicate name is not allowed
+## look up the project list
+* check that "test_project_description" is on the list
+## update project description
+* update the project "test_project_description" and edit the description to be "this is a test project V2"
+## delete project and check
+* delete project "test_project_description" and check that it's not on the list
diff --git a/build/CI/kylin-system-testing/features/step_impl/auto_config/auto_config.py b/build/CI/kylin-system-testing/features/step_impl/auto_config/auto_config.py
new file mode 100644
index 0000000..30ad76f
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/step_impl/auto_config/auto_config.py
@@ -0,0 +1,196 @@
+from getgauge.python import step
+import os
+import json
+import pytest
+from kylin_utils import util
+from kylin_utils import shell
+import re
+import time
+from getgauge.python import Messages
+
+global client
+client = util.setup_instance('kylin_instance.yml')
+
+
+@step("use default configuration on cube <cube_name>")
+def default_config(cube_name):
+ sh_config(set_auto="True", executor_instance="5", instance_strategy="100,2,500,3,1000,4")
+
+ time.sleep(60)
+
+ resp = client.build_segment(start_time="1325376000000",
+ end_time="1388534400000",
+ cube_name=cube_name)
+ job_id = resp['uuid']
+ step_id = job_id + "-01"
+ client.await_job(job_id)
+ resp = client.get_step_output(job_id=job_id, step_id=step_id)
+ output = resp.get('cmd_output')
+
+ check_log(output=output, memorySize="4GB", coresSize="5", memoryOverheadSize="1GB", instancesSize="5", partitionsSize="2")
+
+ client.disable_cube(cube_name)
+
+ time.sleep(30)
+ client.purge_cube(cube_name)
+
+ time.sleep(30)
+
+
+@step("auto configuration with user-defined parameters on cube <cube_name>")
+def user_def_config(cube_name):
+ sh_config(set_auto="True", executor_instance="1", instance_strategy="100,1,500,2,1000,3")
+
+ time.sleep(60)
+
+ resp = client.build_segment(start_time="1325376000000",
+ end_time="1388534400000",
+ cube_name=cube_name)
+
+ job_id = resp['uuid']
+ step_id = job_id + "-01"
+ client.await_job(job_id)
+ resp = client.get_step_output(job_id=job_id, step_id=step_id)
+ output = resp.get('cmd_output')
+
+ check_log(output=output, memorySize="4GB", coresSize="5", memoryOverheadSize="1GB", instancesSize="1", partitionsSize="2")
+
+ client.disable_cube(cube_name)
+
+ time.sleep(30)
+ client.purge_cube(cube_name)
+
+ time.sleep(30)
+
+
+@step("auto configuration with user-defined parameters on <cube_name> and <cube_no_distinct>")
+def user_defined_no_dist(cube_name, cube_no_distinct):
+ sh_config(set_auto="True", executor_instance="2", instance_strategy="100,4,500,6,1000,10")
+
+ time.sleep(60)
+
+ resp = client.build_segment(start_time="1325376000000",
+ end_time="1388534400000",
+ cube_name=cube_name)
+
+ job_id = resp['uuid']
+ step_id = job_id + "-01"
+ client.await_job(job_id)
+ resp = client.get_step_output(job_id=job_id, step_id=step_id)
+ output = resp.get('cmd_output')
+
+ check_log(output=output, memorySize="4GB", coresSize="5", memoryOverheadSize="1GB", instancesSize="2", partitionsSize="2")
+
+ client.disable_cube(cube_name)
+
+ time.sleep(30)
+ client.purge_cube(cube_name)
+
+ time.sleep(30)
+
+ with open(os.path.join('meta_data/auto_config', 'auto_config_no_distinct.json'), 'r') as f:
+ cube_desc_data = json.load(f)['cube_desc_data']
+
+ client.create_cube('learn_kylin', cube_no_distinct, cube_desc_data=cube_desc_data)
+
+ resp = client.build_segment(start_time="1325376000000",
+ end_time="1388534400000",
+ cube_name=cube_no_distinct)
+
+ job_id = resp['uuid']
+ step_id = job_id + "-01"
+ client.await_job(job_id)
+ resp = client.get_step_output(job_id=job_id, step_id=step_id)
+ output = resp.get('cmd_output')
+
+ check_log(output=output, memorySize="1GB", coresSize="1", memoryOverheadSize="512MB", instancesSize="2", partitionsSize="2")
+
+ client.disable_cube(cube_no_distinct)
+
+ time.sleep(30)
+ client.delete_cube(cube_no_distinct)
+
+ time.sleep(30)
+
+
+@step("auto_config and override on cube level on cube <cube_override_conf>")
+def override_on_cube(cube_override_conf):
+ sh_config(set_auto="True", executor_instance="5", instance_strategy="100,2,500,3,1000,4")
+
+ time.sleep(60)
+
+ with open(os.path.join('meta_data/auto_config', 'auto_config_override_conf.json'), 'r') as f:
+ cube_desc_data = json.load(f)['cube_desc_data']
+
+ client.create_cube('learn_kylin', cube_override_conf, cube_desc_data=cube_desc_data)
+
+ resp = client.build_segment(start_time="1325376000000",
+ end_time="1388534400000",
+ cube_name=cube_override_conf)
+
+ job_id = resp['uuid']
+ step_id = job_id + "-01"
+ client.await_job(job_id)
+ resp = client.get_step_output(job_id=job_id, step_id=step_id)
+ output = resp.get('cmd_output')
+
+ memory = re.findall("Override user-defined spark conf, set spark.executor.memory.*", output)
+ cores = re.findall("Override user-defined spark conf, set spark.executor.cores.*", output)
+ memoryOverhead = re.findall("Override user-defined spark conf, set spark.executor.memoryOverhead.*", output)
+ instances = re.findall("Override user-defined spark conf, set spark.executor.instances.*", output)
+ partitions = re.findall("Override user-defined spark conf, set spark.sql.shuffle.partitions.*", output)
+ assert memory[0] == "Override user-defined spark conf, set spark.executor.memory=2G.", \
+ Messages.write_message("expected Override user-defined spark conf, set spark.executor.memory=2G; actually " + memory[0])
+ assert cores[0] == "Override user-defined spark conf, set spark.executor.cores=2.", \
+ Messages.write_message("expected Override user-defined spark conf, set spark.executor.cores=2; actually " + cores[0])
+ assert memoryOverhead[0] == "Override user-defined spark conf, set spark.executor.memoryOverhead=256M.", \
+ Messages.write_message("expected Override user-defined spark conf, set spark.executor.memoryOverhead=256M; actually " + memoryOverhead[0])
+ assert instances[0] == "Override user-defined spark conf, set spark.executor.instances=3.",\
+ Messages.write_message("expected Override user-defined spark conf, set spark.executor.instances=3; actually" + instances[0])
+ assert partitions[0] == "Override user-defined spark conf, set spark.sql.shuffle.partitions=3.", \
+ Messages.write_message("expected Override user-defined spark conf, set spark.sql.shuffle.partitions=3; actually " + partitions[0])
+
+ client.disable_cube(cube_override_conf)
+
+ time.sleep(30)
+ client.delete_cube(cube_override_conf)
+
+ time.sleep(30)
+
+
+def sh_config(set_auto, executor_instance, instance_strategy):
+ sh = util.ssh_shell()
+
+ sh_command = "cd $KYLIN_HOME/conf " \
+ "&& sed -i -r '/kylin.spark-conf.auto.prior/d' kylin.properties" \
+ "&& sed -i -r '/kylin.engine.base-executor-instance/d' kylin.properties" \
+ "&& sed -i -r '/kylin.engine.executor-instance-strategy/d' kylin.properties" \
+ "&& sed -i '$akylin.spark-conf.auto.prior={set_auto}' kylin.properties" \
+ "&& sed -i '$akylin.engine.base-executor-instance={executor_instance}' kylin.properties" \
+ "&& sed -i '$akylin.engine.executor-instance-strategy={instance_strategy}' kylin.properties".format(
+ set_auto=set_auto, executor_instance=executor_instance, instance_strategy=instance_strategy)
+
+ resp = sh.command(sh_command)
+
+ resp = sh.command("cd $KYLIN_HOME/bin"
+ "&& sh kylin.sh stop"
+ "&& sh kylin.sh start")
+
+
+def check_log(output, memorySize, coresSize, memoryOverheadSize, instancesSize, partitionsSize):
+ memory = re.findall("Auto set spark conf: spark.executor.memory = .*", output)
+ cores = re.findall("Auto set spark conf: spark.executor.cores = .*", output)
+ memoryOverhead = re.findall("Auto set spark conf: spark.executor.memoryOverhead = .*", output)
+ instances = re.findall("Auto set spark conf: spark.executor.instances = .*", output)
+ partitions = re.findall("Auto set spark conf: spark.sql.shuffle.partitions = .*", output)
+ assert memory[0] == "Auto set spark conf: spark.executor.memory = {memory}.".format(memory = memorySize), \
+ Messages.write_message("expected "+"Auto set spark conf: spark.executor.memory = {memory}.".format(memory = memorySize)+ ", actually " +memory[0])
+ assert cores[0] == "Auto set spark conf: spark.executor.cores = {cores}.".format(cores = coresSize), \
+ Messages.write_message("expected "+"Auto set spark conf: spark.executor.cores = {cores}.".format(cores = coresSize)+", actually " +cores[0])
+ assert memoryOverhead[0] == "Auto set spark conf: spark.executor.memoryOverhead = {memoryOverhead}.".format(memoryOverhead = memoryOverheadSize), \
+ Messages.write_message("expected "+"Auto set spark conf: spark.executor.memoryOverhead = {memoryOverhead}.".format(memoryOverhead = memoryOverheadSize)+", actually " +memoryOverhead[0])
+ assert instances[0] == "Auto set spark conf: spark.executor.instances = {instances}.".format(instances = instancesSize), \
+ Messages.write_message("expected "+"Auto set spark conf: spark.executor.instances = {instances}.".format(instances = instancesSize)+", actually " +instances[0])
+ assert partitions[0] == "Auto set spark conf: spark.sql.shuffle.partitions = {partitions}.".format(partitions = partitionsSize), \
+ Messages.write_message("expected "+"Auto set spark conf: spark.sql.shuffle.partitions = {partitions}.".format(partitions = partitionsSize)+", actually " +partitions[0])
+
diff --git a/build/CI/kylin-system-testing/features/step_impl/before_suite.py b/build/CI/kylin-system-testing/features/step_impl/before_suite.py
index 00fca21..e0843a2 100644
--- a/build/CI/kylin-system-testing/features/step_impl/before_suite.py
+++ b/build/CI/kylin-system-testing/features/step_impl/before_suite.py
@@ -43,6 +43,8 @@ def create_generic_model_and_cube():
model_desc_data = data.get('model_desc_data')
model_name = model_desc_data.get('name')
+ snowflake_left_incre_model = data.get('snowflake_left_incre_model')
+ left_model_name = snowflake_left_incre_model.get('name')
if not util.if_model_exists(kylin_client=client, model_name=model_name, project=project_name):
resp = client.create_model(project_name=project_name,
@@ -50,6 +52,12 @@ def create_generic_model_and_cube():
model_desc_data=model_desc_data)
assert json.loads(resp['modelDescData'])['name'] == model_name
+ if not util.if_model_exists(kylin_client=client, model_name=left_model_name, project=project_name):
+ resp = client.create_model(project_name=project_name,
+ model_name=left_model_name,
+ model_desc_data=snowflake_left_incre_model)
+ assert json.loads(resp['modelDescData'])['name'] == left_model_name
+
cube_desc_data = data.get('cube_desc_data')
cube_name = cube_desc_data.get('name')
if not util.if_cube_exists(kylin_client=client, cube_name=cube_name, project=project_name):
diff --git a/build/CI/kylin-system-testing/features/step_impl/happy_path/happy_path.py b/build/CI/kylin-system-testing/features/step_impl/happy_path/happy_path.py
new file mode 100644
index 0000000..1d59e05
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/step_impl/happy_path/happy_path.py
@@ -0,0 +1,68 @@
+from getgauge.python import step
+import os
+import json
+import pytest
+from kylin_utils import util
+from kylin_utils import equals
+from kylin_utils import shell
+import re
+import time
+
+
+@step("automated happy path, check query using <sql_file>")
+def happy_path(sql_file):
+ global client
+ client = util.setup_instance('kylin_instance.yml')
+
+ resp1 = client.build_segment(start_time="1325376000000",
+ end_time="1356998400000",
+ cube_name="kylin_sales_cube")
+
+ resp2 = client.build_segment(start_time="1356998400000",
+ end_time="1388620800000",
+ cube_name="kylin_sales_cube")
+
+ job_id1 = resp1['uuid']
+ job_id2 = resp2['uuid']
+ client.await_job(job_id1)
+ client.await_job(job_id2)
+
+ resp = client.merge_segment(cube_name="kylin_sales_cube",
+ start_time="1325376000000",
+ end_time="1388620800000")
+
+ job_id = resp['uuid']
+ client.await_job(job_id)
+
+ resp = client.refresh_segment(cube_name="kylin_sales_cube",
+ start_time="1325376000000",
+ end_time="1388620800000")
+ job_id = resp['uuid']
+ client.await_job(job_id)
+
+ with open(sql_file, 'r', encoding='utf8') as sql:
+ sql = sql.read()
+
+ equals.compare_sql_result(sql=sql, project='learn_kylin', kylin_client=client, expected_result=None)
+
+ resp = client.disable_cube(cube_name="kylin_sales_cube")
+ assert resp['status'] == 'DISABLED'
+
+ time.sleep(10)
+
+ client.purge_cube(cube_name="kylin_sales_cube")
+ time.sleep(30)
+
+ resp = client.get_cube_instance(cube_name="kylin_sales_cube")
+
+ assert len(resp['segments']) == 0
+
+
+
+
+
+
+
+
+
+
diff --git a/build/CI/kylin-system-testing/features/step_impl/project_model/model.py b/build/CI/kylin-system-testing/features/step_impl/project_model/model.py
new file mode 100644
index 0000000..f159d93
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/step_impl/project_model/model.py
@@ -0,0 +1,28 @@
+from getgauge.python import step
+import os
+import json
+import pytest
+from kylin_utils import util
+
+global client
+client = util.setup_instance('kylin_instance.yml')
+
+
+@step("in project <project_name>, clone model <model_name> and name it <clone_name>")
+def check_model_clone(project_name, model_name, clone_name):
+ clone = client.clone_model(project_name, model_name, clone_name)
+ assert util.if_model_exists(kylin_client=client, model_name=clone_name, project=project_name) == 1
+ model_desc = client.list_model_desc(project_name, model_name)
+ clone_model_desc = client.list_model_desc(project_name, clone_name)
+ check_list = ['fact_table', 'lookups', 'dimensions', 'metrics', 'filter_condition', 'partition_desc']
+ for i in range(len(check_list)):
+ assert model_desc[0][check_list[i]] == clone_model_desc[0][check_list[i]]
+
+
+@step("again, in project <project_name>, clone model <model_name> and name it <clone_name>")
+def check_clone_duplicated(project_name, model_name, clone_name):
+ with pytest.raises(Exception, match=r'Model name .* is duplicated, could not be created.'):
+ clone = client.clone_model(project_name, model_name, clone_name)
+
+
+
diff --git a/build/CI/kylin-system-testing/features/step_impl/project_model/project.py b/build/CI/kylin-system-testing/features/step_impl/project_model/project.py
new file mode 100644
index 0000000..61a088f
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/step_impl/project_model/project.py
@@ -0,0 +1,46 @@
+from getgauge.python import step
+import os
+import json
+import pytest
+from kylin_utils import util
+
+global client
+client = util.setup_instance('kylin_instance.yml')
+
+@step("create a project <project_name> with description <project_description> and check that duplicate name is not allowed")
+def create_duplicate_project(project_name, project_description):
+ client.create_project(project_name, description=project_description)
+ with pytest.raises(Exception, match=r'The project named .* already exists'):
+ client.create_project(project_name, description=project_description)
+
+
+@step("check that <project_name> is on the list")
+def check_project(project_name):
+
+ assert util.if_project_exists(kylin_client=client, project=project_name) == 1
+
+
+@step("update the project <project_name> and edit the description to be <project_description>")
+def update_project_description(project_name, project_description):
+ update = client.update_project(project_name, description=project_description)
+ resp = client.list_projects()
+ update = False
+ for i in range(len(resp)):
+ update = update | ((resp[i]['name'] == project_name) & (resp[i]['description'] == project_description))
+ assert update is True
+
+
+@step("delete project <project_name> and check that it's not on the list")
+def check_delete_project(project_name):
+ delete = client.delete_project(project_name)
+ resp = client.list_projects()
+ exist = False
+ for i in range(len(resp)):
+ exist = exist | (resp[i]['name'] == project_name)
+ assert exist is False
+
+
+
+
+
+
diff --git a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml b/build/CI/kylin-system-testing/kylin_instances/kylin_host.yml
similarity index 88%
copy from build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml
copy to build/CI/kylin-system-testing/kylin_instances/kylin_host.yml
index 5454a41..c75bbf5 100644
--- a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml
+++ b/build/CI/kylin-system-testing/kylin_instances/kylin_host.yml
@@ -14,9 +14,6 @@
# limitations under the License.
---
-# All mode
-- host: localhost
- port: 7070
- version: 4.x
- hadoop_platform: HDP2.4
- deploy_mode: ALL
\ No newline at end of file
+- host: kylin-all
+ username: root
+ password: hadoop
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml b/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml
index 5454a41..1aa1a08 100644
--- a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml
+++ b/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml
@@ -15,7 +15,7 @@
---
# All mode
-- host: localhost
+- host: kylin-all
port: 7070
version: 4.x
hadoop_platform: HDP2.4
diff --git a/build/CI/kylin-system-testing/kylin_utils/equals.py b/build/CI/kylin-system-testing/kylin_utils/equals.py
index 2cceff1..e352736 100644
--- a/build/CI/kylin-system-testing/kylin_utils/equals.py
+++ b/build/CI/kylin-system-testing/kylin_utils/equals.py
@@ -241,7 +241,7 @@ def compare_sql_result(sql, project, kylin_client, compare_level="data_set", cub
kylin_resp = kylin_client.execute_query(cube_name=cube,
project_name=project,
sql=sql)
- assert kylin_resp.get('isException') is False, 'Thown Exception when execute ' + sql
+ assert kylin_resp.get('isException') is False, 'Thrown Exception when execute ' + sql
pushdown_resp = kylin_client.execute_query(project_name=pushdown_project, sql=sql)
assert pushdown_resp.get('isException') is False
diff --git a/build/CI/kylin-system-testing/kylin_utils/shell.py b/build/CI/kylin-system-testing/kylin_utils/shell.py
index 72b734a..31af6af 100644
--- a/build/CI/kylin-system-testing/kylin_utils/shell.py
+++ b/build/CI/kylin-system-testing/kylin_utils/shell.py
@@ -135,8 +135,3 @@ def exec(script): # pylint: disable=redefined-builtin
def shell():
return Bash()
-
-
-if __name__ == '__main__':
- sh = sshshell('10.1.3.94', username='root', password='hadoop')
- print(sh.command('pwd'))
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/kylin_utils/util.py b/build/CI/kylin-system-testing/kylin_utils/util.py
index f29e034..44d3e4a 100644
--- a/build/CI/kylin-system-testing/kylin_utils/util.py
+++ b/build/CI/kylin-system-testing/kylin_utils/util.py
@@ -18,7 +18,7 @@
from selenium import webdriver
from yaml import load, loader
import os
-
+from kylin_utils import shell
from kylin_utils import kylin
@@ -79,3 +79,12 @@ def if_model_exists(kylin_client, model_name, project):
if len(resp) == 1:
exists = 1
return exists
+
+def ssh_shell(config_file='kylin_host.yml'):
+ instances_file = os.path.join('kylin_instances/', config_file)
+ stream = open(instances_file, 'r')
+ for item in load(stream, Loader=loader.SafeLoader):
+ host = item['host']
+ username = item['username']
+ password = item['password']
+ return shell.SSHShell(host=host, username=username, password=password)