You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/11/08 08:13:12 UTC
[kylin] 12/13: KYLIN-4801 Clean up and add test sql
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 97cebbdca78911d505b63ad16ea5df825c951612
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Tue Nov 3 18:03:10 2020 +0800
KYLIN-4801 Clean up and add test sql
---
.../data/release_test_0001.json | 626 ---------------------
.../features/specs/generic_test.spec | 64 ---
.../features/specs/query/query.spec | 22 +
.../features/step_impl/before_suite.py | 10 +-
.../features/step_impl/generic_test_step.py | 15 +-
.../features/step_impl/query/query.py | 40 ++
.../kylin_instances/kylin_instance.yml | 2 +-
.../CI/kylin-system-testing/kylin_utils/equals.py | 11 +-
.../generic_desc_data/generic_desc_data_3x.json | 0
.../generic_desc_data/generic_desc_data_4x.json | 2 +-
.../query/sql/sql_test/sql1.sql | 26 +
.../query/sql_result/sql_test/sql1.json | 6 +
build/CI/run-ci.sh | 14 +-
13 files changed, 126 insertions(+), 712 deletions(-)
diff --git a/build/CI/kylin-system-testing/data/release_test_0001.json b/build/CI/kylin-system-testing/data/release_test_0001.json
deleted file mode 100644
index 0b5558a..0000000
--- a/build/CI/kylin-system-testing/data/release_test_0001.json
+++ /dev/null
@@ -1,626 +0,0 @@
-{
- "load_table_list":
- "DEFAULT.KYLIN_SALES,DEFAULT.KYLIN_CAL_DT,DEFAULT.KYLIN_CATEGORY_GROUPINGS,DEFAULT.KYLIN_ACCOUNT,DEFAULT.KYLIN_COUNTRY",
-
- "model_desc_data":
- {
- "uuid": "0928468a-9fab-4185-9a14-6f2e7c74823f",
- "last_modified": 0,
- "version": "3.0.0.20500",
- "name": "release_test_0001_model",
- "owner": null,
- "is_draft": false,
- "description": "",
- "fact_table": "DEFAULT.KYLIN_SALES",
- "lookups": [
- {
- "table": "DEFAULT.KYLIN_CAL_DT",
- "kind": "LOOKUP",
- "alias": "KYLIN_CAL_DT",
- "join": {
- "type": "inner",
- "primary_key": [
- "KYLIN_CAL_DT.CAL_DT"
- ],
- "foreign_key": [
- "KYLIN_SALES.PART_DT"
- ]
- }
- },
- {
- "table": "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "kind": "LOOKUP",
- "alias": "KYLIN_CATEGORY_GROUPINGS",
- "join": {
- "type": "inner",
- "primary_key": [
- "KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID",
- "KYLIN_CATEGORY_GROUPINGS.SITE_ID"
- ],
- "foreign_key": [
- "KYLIN_SALES.LEAF_CATEG_ID",
- "KYLIN_SALES.LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "DEFAULT.KYLIN_ACCOUNT",
- "kind": "LOOKUP",
- "alias": "BUYER_ACCOUNT",
- "join": {
- "type": "inner",
- "primary_key": [
- "BUYER_ACCOUNT.ACCOUNT_ID"
- ],
- "foreign_key": [
- "KYLIN_SALES.BUYER_ID"
- ]
- }
- },
- {
- "table": "DEFAULT.KYLIN_ACCOUNT",
- "kind": "LOOKUP",
- "alias": "SELLER_ACCOUNT",
- "join": {
- "type": "inner",
- "primary_key": [
- "SELLER_ACCOUNT.ACCOUNT_ID"
- ],
- "foreign_key": [
- "KYLIN_SALES.SELLER_ID"
- ]
- }
- },
- {
- "table": "DEFAULT.KYLIN_COUNTRY",
- "kind": "LOOKUP",
- "alias": "BUYER_COUNTRY",
- "join": {
- "type": "inner",
- "primary_key": [
- "BUYER_COUNTRY.COUNTRY"
- ],
- "foreign_key": [
- "BUYER_ACCOUNT.ACCOUNT_COUNTRY"
- ]
- }
- },
- {
- "table": "DEFAULT.KYLIN_COUNTRY",
- "kind": "LOOKUP",
- "alias": "SELLER_COUNTRY",
- "join": {
- "type": "inner",
- "primary_key": [
- "SELLER_COUNTRY.COUNTRY"
- ],
- "foreign_key": [
- "SELLER_ACCOUNT.ACCOUNT_COUNTRY"
- ]
- }
- }
- ],
- "dimensions": [
- {
- "table": "KYLIN_SALES",
- "columns": [
- "TRANS_ID",
- "SELLER_ID",
- "BUYER_ID",
- "PART_DT",
- "LEAF_CATEG_ID",
- "LSTG_FORMAT_NAME",
- "LSTG_SITE_ID",
- "OPS_USER_ID",
- "OPS_REGION"
- ]
- },
- {
- "table": "KYLIN_CAL_DT",
- "columns": [
- "CAL_DT",
- "WEEK_BEG_DT",
- "MONTH_BEG_DT",
- "YEAR_BEG_DT"
- ]
- },
- {
- "table": "KYLIN_CATEGORY_GROUPINGS",
- "columns": [
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "META_CATEG_NAME",
- "CATEG_LVL2_NAME",
- "CATEG_LVL3_NAME",
- "LEAF_CATEG_ID",
- "SITE_ID"
- ]
- },
- {
- "table": "BUYER_ACCOUNT",
- "columns": [
- "ACCOUNT_ID",
- "ACCOUNT_BUYER_LEVEL",
- "ACCOUNT_SELLER_LEVEL",
- "ACCOUNT_COUNTRY",
- "ACCOUNT_CONTACT"
- ]
- },
- {
- "table": "SELLER_ACCOUNT",
- "columns": [
- "ACCOUNT_ID",
- "ACCOUNT_BUYER_LEVEL",
- "ACCOUNT_SELLER_LEVEL",
- "ACCOUNT_COUNTRY",
- "ACCOUNT_CONTACT"
- ]
- },
- {
- "table": "BUYER_COUNTRY",
- "columns": [
- "COUNTRY",
- "NAME"
- ]
- },
- {
- "table": "SELLER_COUNTRY",
- "columns": [
- "COUNTRY",
- "NAME"
- ]
- }
- ],
- "metrics": [
- "KYLIN_SALES.PRICE",
- "KYLIN_SALES.ITEM_COUNT"
- ],
- "filter_condition": "",
- "partition_desc": {
- "partition_date_column": "KYLIN_SALES.PART_DT",
- "partition_time_column": null,
- "partition_date_start": 0,
- "partition_date_format": "yyyy-MM-dd HH:mm:ss",
- "partition_time_format": "HH:mm:ss",
- "partition_type": "APPEND",
- "partition_condition_builder": "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
- },
- "capacity": "MEDIUM",
- "projectName": null
- },
- "cube_desc_data":
- {
- "uuid": "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "last_modified": 0,
- "version": "3.0.0.20500",
- "name": "release_test_0001_cube",
- "is_draft": false,
- "model_name": "release_test_0001_model",
- "description": "",
- "null_string": null,
- "dimensions": [
- {
- "name": "TRANS_ID",
- "table": "KYLIN_SALES",
- "column": "TRANS_ID",
- "derived": null
- },
- {
- "name": "YEAR_BEG_DT",
- "table": "KYLIN_CAL_DT",
- "column": null,
- "derived": [
- "YEAR_BEG_DT"
- ]
- },
- {
- "name": "MONTH_BEG_DT",
- "table": "KYLIN_CAL_DT",
- "column": null,
- "derived": [
- "MONTH_BEG_DT"
- ]
- },
- {
- "name": "WEEK_BEG_DT",
- "table": "KYLIN_CAL_DT",
- "column": null,
- "derived": [
- "WEEK_BEG_DT"
- ]
- },
- {
- "name": "USER_DEFINED_FIELD1",
- "table": "KYLIN_CATEGORY_GROUPINGS",
- "column": null,
- "derived": [
- "USER_DEFINED_FIELD1"
- ]
- },
- {
- "name": "USER_DEFINED_FIELD3",
- "table": "KYLIN_CATEGORY_GROUPINGS",
- "column": null,
- "derived": [
- "USER_DEFINED_FIELD3"
- ]
- },
- {
- "name": "META_CATEG_NAME",
- "table": "KYLIN_CATEGORY_GROUPINGS",
- "column": "META_CATEG_NAME",
- "derived": null
- },
- {
- "name": "CATEG_LVL2_NAME",
- "table": "KYLIN_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL2_NAME",
- "derived": null
- },
- {
- "name": "CATEG_LVL3_NAME",
- "table": "KYLIN_CATEGORY_GROUPINGS",
- "column": "CATEG_LVL3_NAME",
- "derived": null
- },
- {
- "name": "LSTG_FORMAT_NAME",
- "table": "KYLIN_SALES",
- "column": "LSTG_FORMAT_NAME",
- "derived": null
- },
- {
- "name": "SELLER_ID",
- "table": "KYLIN_SALES",
- "column": "SELLER_ID",
- "derived": null
- },
- {
- "name": "BUYER_ID",
- "table": "KYLIN_SALES",
- "column": "BUYER_ID",
- "derived": null
- },
- {
- "name": "ACCOUNT_BUYER_LEVEL",
- "table": "BUYER_ACCOUNT",
- "column": "ACCOUNT_BUYER_LEVEL",
- "derived": null
- },
- {
- "name": "ACCOUNT_SELLER_LEVEL",
- "table": "SELLER_ACCOUNT",
- "column": "ACCOUNT_SELLER_LEVEL",
- "derived": null
- },
- {
- "name": "BUYER_COUNTRY",
- "table": "BUYER_ACCOUNT",
- "column": "ACCOUNT_COUNTRY",
- "derived": null
- },
- {
- "name": "SELLER_COUNTRY",
- "table": "SELLER_ACCOUNT",
- "column": "ACCOUNT_COUNTRY",
- "derived": null
- },
- {
- "name": "BUYER_COUNTRY_NAME",
- "table": "BUYER_COUNTRY",
- "column": "NAME",
- "derived": null
- },
- {
- "name": "SELLER_COUNTRY_NAME",
- "table": "SELLER_COUNTRY",
- "column": "NAME",
- "derived": null
- },
- {
- "name": "OPS_USER_ID",
- "table": "KYLIN_SALES",
- "column": "OPS_USER_ID",
- "derived": null
- },
- {
- "name": "OPS_REGION",
- "table": "KYLIN_SALES",
- "column": "OPS_REGION",
- "derived": null
- }
- ],
- "measures": [
- {
- "name": "GMV_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "KYLIN_SALES.PRICE"
- },
- "returntype": "decimal(19,4)"
- }
- },
- {
- "name": "BUYER_LEVEL_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL"
- },
- "returntype": "bigint"
- }
- },
- {
- "name": "SELLER_LEVEL_SUM",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"
- },
- "returntype": "bigint"
- }
- },
- {
- "name": "TRANS_CNT",
- "function": {
- "expression": "COUNT",
- "parameter": {
- "type": "constant",
- "value": "1"
- },
- "returntype": "bigint"
- }
- },
- {
- "name": "SELLER_CNT_HLL",
- "function": {
- "expression": "COUNT_DISTINCT",
- "parameter": {
- "type": "column",
- "value": "KYLIN_SALES.SELLER_ID"
- },
- "returntype": "hllc(10)"
- }
- },
- {
- "name": "TOP_SELLER",
- "function": {
- "expression": "TOP_N",
- "parameter": {
- "type": "column",
- "value": "KYLIN_SALES.PRICE",
- "next_parameter": {
- "type": "column",
- "value": "KYLIN_SALES.SELLER_ID"
- }
- },
- "returntype": "topn(100)",
- "configuration": {
- "topn.encoding.KYLIN_SALES.SELLER_ID": "dict",
- "topn.encoding_version.KYLIN_SALES.SELLER_ID": "1"
- }
- }
- }
- ],
- "rowkey": {
- "rowkey_columns": [
- {
- "column": "KYLIN_SALES.BUYER_ID",
- "encoding": "integer:4",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_SALES.SELLER_ID",
- "encoding": "integer:4",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_SALES.TRANS_ID",
- "encoding": "integer:4",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_SALES.PART_DT",
- "encoding": "date",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_SALES.LEAF_CATEG_ID",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "BUYER_COUNTRY.NAME",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "SELLER_COUNTRY.NAME",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_SALES.LSTG_FORMAT_NAME",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_SALES.LSTG_SITE_ID",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_SALES.OPS_USER_ID",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- },
- {
- "column": "KYLIN_SALES.OPS_REGION",
- "encoding": "dict",
- "encoding_version": 1,
- "isShardBy": false
- }
- ]
- },
- "hbase_mapping": {
- "column_family": [
- {
- "name": "F1",
- "columns": [
- {
- "qualifier": "M",
- "measure_refs": [
- "GMV_SUM",
- "BUYER_LEVEL_SUM",
- "SELLER_LEVEL_SUM",
- "TRANS_CNT"
- ]
- }
- ]
- },
- {
- "name": "F2",
- "columns": [
- {
- "qualifier": "M",
- "measure_refs": [
- "SELLER_CNT_HLL",
- "TOP_SELLER"
- ]
- }
- ]
- }
- ]
- },
- "aggregation_groups": [
- {
- "includes": [
- "KYLIN_SALES.PART_DT",
- "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME",
- "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
- "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
- "KYLIN_SALES.LEAF_CATEG_ID",
- "KYLIN_SALES.LSTG_FORMAT_NAME",
- "KYLIN_SALES.LSTG_SITE_ID",
- "KYLIN_SALES.OPS_USER_ID",
- "KYLIN_SALES.OPS_REGION",
- "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
- "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
- "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
- "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
- "BUYER_COUNTRY.NAME",
- "SELLER_COUNTRY.NAME"
- ],
- "select_rule": {
- "hierarchy_dims": [
- [
- "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME",
- "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
- "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
- "KYLIN_SALES.LEAF_CATEG_ID"
- ]
- ],
- "mandatory_dims": [
- "KYLIN_SALES.PART_DT"
- ],
- "joint_dims": [
- [
- "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
- "BUYER_COUNTRY.NAME"
- ],
- [
- "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
- "SELLER_COUNTRY.NAME"
- ],
- [
- "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
- "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL"
- ],
- [
- "KYLIN_SALES.LSTG_FORMAT_NAME",
- "KYLIN_SALES.LSTG_SITE_ID"
- ],
- [
- "KYLIN_SALES.OPS_USER_ID",
- "KYLIN_SALES.OPS_REGION"
- ]
- ]
- }
- }
- ],
- "signature": null,
- "notify_list": [],
- "status_need_notify": [],
- "partition_date_start": 0,
- "partition_date_end": 3153600000000,
- "auto_merge_time_ranges": [],
- "volatile_range": 0,
- "retention_range": 0,
- "engine_type": 2,
- "storage_type": 2,
- "override_kylin_properties": {
- "kylin.cube.aggrgroup.is-mandatory-only-valid": "true",
- "kylin.engine.spark.rdd-partition-cut-mb": "500"
- },
- "cuboid_black_list": [],
- "parent_forward": 3,
- "mandatory_dimension_set_list": [],
- "snapshot_table_desc_list": []
- }
-}
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/features/specs/generic_test.spec b/build/CI/kylin-system-testing/features/specs/generic_test.spec
deleted file mode 100644
index d37e236..0000000
--- a/build/CI/kylin-system-testing/features/specs/generic_test.spec
+++ /dev/null
@@ -1,64 +0,0 @@
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-# Kylin Release Test
-Tags:3.x
-## Prepare env
-* Get kylin instance
-
-* prepare data file from "release_test_0001.json"
-
-* Create project "release_test_0001_project" and load table "load_table_list"
-
-
-## MR engine
-
-* Create model with "model_desc_data" in "release_test_0001_project"
-
-* Create cube with "cube_desc_data" in "release_test_0001_project", cube name is "release_test_0001_cube"
-
-* Build segment from "1325347200000" to "1356969600000" in "release_test_0001_cube"
-
-* Build segment from "1356969600000" to "1391011200000" in "release_test_0001_cube"
-
-* Merge cube "release_test_0001_cube" segment from "1325347200000" to "1391011200000"
-
-
-SPARK engine
-
-Clone cube "release_test_0001_cube" and name it "kylin_spark_cube" in "release_test_0001_project", modify build engine to "SPARK"
-
-Build segment from "1325347200000" to "1356969600000" in "kylin_spark_cube"
-
-Build segment from "1356969600000" to "1391011200000" in "kylin_spark_cube"
-
-Merge cube "kylin_spark_cube" segment from "1325347200000" to "1391011200000"
-
-
-## Query cube and pushdown
-
-* Query SQL "select count(*) from kylin_sales" and specify "release_test_0001_cube" cube to query in "release_test_0001_project", compare result with "10000"
-
-Query SQL "select count(*) from kylin_sales" and specify "kylin_spark_cube" cube to query in "release_test_0001_project", compare result with "10000"
-
-* Disable cube "release_test_0001_cube"
-
-Disable cube "kylin_spark_cube"
-
-* Query SQL "select count(*) from kylin_sales" in "release_test_0001_project" and pushdown, compare result with "10000"
-
-
-
diff --git a/build/CI/kylin-system-testing/features/specs/query/query.spec b/build/CI/kylin-system-testing/features/specs/query/query.spec
new file mode 100644
index 0000000..8cd3a6f
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/specs/query/query.spec
@@ -0,0 +1,22 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+# Kylin SQL test
+Tags:4.x
+
+## Query sql
+
+* Query all SQL file in directory "query/sql/sql_test/" in project "generic_test_project", compare result with hive pushdown result and compare metrics info with sql_result json file in "query/sql_result/sql_test/"
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/features/step_impl/before_suite.py b/build/CI/kylin-system-testing/features/step_impl/before_suite.py
index 3cd86ca..d1cd3fd 100644
--- a/build/CI/kylin-system-testing/features/step_impl/before_suite.py
+++ b/build/CI/kylin-system-testing/features/step_impl/before_suite.py
@@ -26,10 +26,10 @@ from kylin_utils import util
def create_generic_model_and_cube():
client = util.setup_instance('kylin_instance.yml')
if client.version == '3.x':
- with open(os.path.join('data/generic_desc_data', 'generic_desc_data_3x.json'), 'r') as f:
+ with open(os.path.join('meta_data/generic_desc_data', 'generic_desc_data_3x.json'), 'r') as f:
data = json.load(f)
elif client.version == '4.x':
- with open(os.path.join('data/generic_desc_data', 'generic_desc_data_4x.json'), 'r') as f:
+ with open(os.path.join('meta_data/generic_desc_data', 'generic_desc_data_4x.json'), 'r') as f:
data = json.load(f)
project_name = client.generic_project
@@ -56,6 +56,6 @@ def create_generic_model_and_cube():
cube_name=cube_name,
cube_desc_data=cube_desc_data)
assert json.loads(resp['cubeDescData'])['name'] == cube_name
- if client.get_cube_instance(cube_name=cube_name).get('status') != 'READY':
- resp = client.full_build_cube(cube_name=cube_name)
- assert client.await_job_finished(job_id=resp['uuid'], waiting_time=20)
+ if client.get_cube_instance(cube_name=cube_name).get('status') != 'READY' and len(client.list_jobs(project_name=project_name, job_search_mode='CUBING_ONLY')) == 0:
+ client.full_build_cube(cube_name=cube_name)
+ assert client.await_all_jobs(project_name=project_name)
diff --git a/build/CI/kylin-system-testing/features/step_impl/generic_test_step.py b/build/CI/kylin-system-testing/features/step_impl/generic_test_step.py
index 0aabb98..cea47c1 100644
--- a/build/CI/kylin-system-testing/features/step_impl/generic_test_step.py
+++ b/build/CI/kylin-system-testing/features/step_impl/generic_test_step.py
@@ -28,10 +28,10 @@ def get_kylin_instance_with_config_file():
client = util.setup_instance('kylin_instance.yml')
-@step("prepare data file from <release_test_0001.json>")
-def prepare_data_file_from(file_name):
+@step("prepare data file from release_test_0001.json")
+def prepare_data_file_from_data(file_name):
global data
- with open(os.path.join('data', file_name), 'r') as f:
+ with open(os.path.join('meta_data', file_name), 'r') as f:
data = json.load(f)
@@ -54,7 +54,7 @@ def create_model_step(model_desc, project):
assert json.loads(resp['modelDescData'])['name'] == model_name
-@step("Create cube with <cube_desc> in <prpject>, cube name is <cube_name>")
+@step("Create cube with <cube_desc> in <project>, cube name is <cube_name>")
def create_cube_step(cube_desc, project, cube_name):
resp = client.create_cube(project_name=project,
cube_name=cube_name,
@@ -113,3 +113,10 @@ def query_pushdown_step(sql, project, result):
assert resp.get('cube') == ''
assert resp.get('pushDown') is True
+
+@step("Query all SQL file in directory <directory>, compare result with hive pushdown result")
+def query_sql_file_and_compare(directory):
+ sql_directory = os.listdir(directory)
+ for sql_file in sql_directory:
+ sql = open(sql_file, 'r', encoding='utf8')
+ sqltxt = sql.readlines()
diff --git a/build/CI/kylin-system-testing/features/step_impl/query/query.py b/build/CI/kylin-system-testing/features/step_impl/query/query.py
new file mode 100644
index 0000000..55f5597
--- /dev/null
+++ b/build/CI/kylin-system-testing/features/step_impl/query/query.py
@@ -0,0 +1,40 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+
+from getgauge.python import step
+
+from kylin_utils import util
+from kylin_utils import equals
+
+
+@step("Query all SQL file in directory <sql_directory> in project <project_name>, compare result with hive pushdown result and compare metrics info with sql_result json file in <sql_result_directory>")
+def query_sql_file_and_compare(sql_directory, project_name, sql_result_directory):
+ sql_directory_list = os.listdir(sql_directory)
+ for sql_file_name in sql_directory_list:
+ with open(sql_directory + sql_file_name, 'r', encoding='utf8') as sql_file:
+ sql = sql_file.read()
+
+ client = util.setup_instance('kylin_instance.yml')
+ with open(sql_result_directory + sql_file_name.split(".")[0] + '.json', 'r', encoding='utf8') as expected_result_file:
+ expected_result = json.loads(expected_result_file.read())
+ equals.compare_sql_result(sql=sql, project=project_name, kylin_client=client, expected_result=expected_result)
+
+
+
diff --git a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml b/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml
index 501428f..5454a41 100644
--- a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml
+++ b/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml
@@ -17,6 +17,6 @@
# All mode
- host: localhost
port: 7070
- version: 3.x
+ version: 4.x
hadoop_platform: HDP2.4
deploy_mode: ALL
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/kylin_utils/equals.py b/build/CI/kylin-system-testing/kylin_utils/equals.py
index 9d44aaf..6c990d4 100644
--- a/build/CI/kylin-system-testing/kylin_utils/equals.py
+++ b/build/CI/kylin-system-testing/kylin_utils/equals.py
@@ -197,7 +197,7 @@ def dataset_equals(expect,
return True
-def compare_sql_result(sql, project, kylin_client, cube=None):
+def compare_sql_result(sql, project, kylin_client, cube=None, expected_result=None):
pushdown_project = kylin_client.pushdown_project
if not util.if_project_exists(kylin_client=kylin_client, project=pushdown_project):
kylin_client.create_project(project_name=pushdown_project)
@@ -218,4 +218,11 @@ def compare_sql_result(sql, project, kylin_client, cube=None):
pushdown_resp = kylin_client.execute_query(project_name=pushdown_project, sql=sql)
assert pushdown_resp.get('isException') is False
- assert query_result_equals(kylin_resp, pushdown_resp)
\ No newline at end of file
+ assert query_result_equals(kylin_resp, pushdown_resp)
+
+ if expected_result is not None:
+ print(kylin_resp.get("totalScanCount"))
+ assert expected_result.get("totalScanCount") == kylin_resp.get("totalScanCount")
+ assert expected_result.get("totalScanBytes") == kylin_resp.get("totalScanBytes")
+ assert expected_result.get("totalScanFiles") == kylin_resp.get("totalScanFiles")
+ assert expected_result.get("pushDown") == kylin_resp.get("pushDown")
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_3x.json b/build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_3x.json
similarity index 100%
rename from build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_3x.json
rename to build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_3x.json
diff --git a/build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_4x.json b/build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_4x.json
similarity index 99%
rename from build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_4x.json
rename to build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_4x.json
index 8d533b5..4055cf4 100644
--- a/build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_4x.json
+++ b/build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_4x.json
@@ -6,7 +6,7 @@
{
"uuid": "0928468a-9fab-4185-9a14-6f2e7c74823f",
"last_modified": 0,
- "version": "3.0.0.20500",
+ "version": "4.0",
"name": "generic_test_model",
"owner": null,
"is_draft": false,
diff --git a/build/CI/kylin-system-testing/query/sql/sql_test/sql1.sql b/build/CI/kylin-system-testing/query/sql/sql_test/sql1.sql
new file mode 100644
index 0000000..9d5e678
--- /dev/null
+++ b/build/CI/kylin-system-testing/query/sql/sql_test/sql1.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT cal_dt ,sum(price) AS sum_price
+FROM
+(SELECT kylin_cal_dt.cal_dt, kylin_sales.price
+FROM kylin_sales INNER JOIN kylin_cal_dt AS kylin_cal_dt
+ON kylin_sales.part_dt = kylin_cal_dt.cal_dt
+INNER JOIN kylin_category_groupings ON kylin_sales.leaf_categ_id = kylin_category_groupings.leaf_categ_id
+AND kylin_sales.lstg_site_id = kylin_category_groupings.site_id) t
+GROUP BY cal_dt;
\ No newline at end of file
diff --git a/build/CI/kylin-system-testing/query/sql_result/sql_test/sql1.json b/build/CI/kylin-system-testing/query/sql_result/sql_test/sql1.json
new file mode 100644
index 0000000..3c2ec22
--- /dev/null
+++ b/build/CI/kylin-system-testing/query/sql_result/sql_test/sql1.json
@@ -0,0 +1,6 @@
+{
+ "totalScanCount":7349,
+ "totalScanBytes":229078,
+ "totalScanFiles":2,
+ "pushDown": false
+}
\ No newline at end of file
diff --git a/build/CI/run-ci.sh b/build/CI/run-ci.sh
index 41a4bb6..acbb2c7 100644
--- a/build/CI/run-ci.sh
+++ b/build/CI/run-ci.sh
@@ -38,7 +38,7 @@ pwd
# 1. Package kylin
if [[ -z $binary_file ]]; then
cd dev-support/build-release
- bash -x package.sh
+ bash -x packaging.sh
cd -
fi
@@ -55,14 +55,10 @@ mkdir kylin-job
cp -r apache-kylin-bin/* kylin-all
cat > kylin-all/conf/kylin.properties <<EOL
+kylin.metadata.url=kylin_metadata@jdbc,url=jdbc:mysql://metastore-db:3306/metastore,username=kylin,password=kylin,maxActive=10,maxIdle=10
+kylin.env.zookeeper-connect-string=write-zookeeper:2181
kylin.job.scheduler.default=100
-kylin.server.self-discovery-enabled=true
-kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl
-kylin.query.pushdown.update-enabled=false
-kylin.query.pushdown.jdbc.url=jdbc:hive2://write-hive-server:10000/default
-kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver
-kylin.query.pushdown.jdbc.username=hive
-kylin.query.pushdown.jdbc.password=
+kylin.query.pushdown.runner-class-name=org.apache.kylin.query.pushdown.PushDownRunnerSparkImpl
EOL
#cp -r apache-kylin-bin/* kylin-query
@@ -123,7 +119,7 @@ sleep ${AWAIT_SECOND}
cd build/CI/kylin-system-testing
pip install -r requirements.txt
-gauge run --tags 3.x
+gauge run --tags 4.x
cd -
echo "Please check build/CI/kylin-system-testing/reports/html-report/index.html for reports."