You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/03/24 10:58:25 UTC
[2/2] kylin git commit: KYLIN-1531 Add smoke test scripts
KYLIN-1531 Add smoke test scripts
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1411f890
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1411f890
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1411f890
Branch: refs/heads/master
Commit: 1411f8906944f8714289638abf741975d1f1e7d7
Parents: d2476f3
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 24 17:57:32 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Mar 24 17:58:32 2016 +0800
----------------------------------------------------------------------
build/smoke-test/smoke-test.sh | 76 +++++++++++++++++++++++++++
build/smoke-test/sql/sql1.json | 38 ++++++++++++++
build/smoke-test/sql/sql1.sql | 1 +
build/smoke-test/testBuildCube.py | 94 ++++++++++++++++++++++++++++++++++
build/smoke-test/testQuery.py | 62 ++++++++++++++++++++++
5 files changed, 271 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1411f890/build/smoke-test/smoke-test.sh
----------------------------------------------------------------------
diff --git a/build/smoke-test/smoke-test.sh b/build/smoke-test/smoke-test.sh
new file mode 100755
index 0000000..5150297
--- /dev/null
+++ b/build/smoke-test/smoke-test.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# PLEASE BE NOTICED
+#
+# This tool will test some main functionality of kylin binary package, such as metadata, cubing and query with rest APIs.
+# And this tool will use default port(7070) and default metastore name(kylin_metastore) in hdp sandbox. So those resources
+# will be overridden in the test.
+#
+# This tool accepts two parameters, the first is tar package of kylin, the second is target path for decompress, which
+# will contain KYLIN_HOME.
+#
+# Usage: bash smoke-test.sh ../dist/apache-kylin-1.5.0-bin.tar.gz ../dist/apache-kylin-1.5.0-bin
+#
+# The process of smoke test is based on sample metadata & data.
+# 1. run sample.sh to load sample data
+# 2. use rest API to build cube
+# 3. use rest API to execute some SQL statements, which locates under sql directory
+# 4. compare query result with result file under sql directory
+
+PKG_PATH=$1
+TARGET_PATH=$2
+
+cd $(dirname ${0})/..
+dir=`pwd`
+mkdir -p ${TARGET_PATH}
+
+# Setup stage
+KYLIN_PID=`cat "${TARGET_PATH}/*kylin*/pid"`
+if [ -n "${KYLIN_PID}" ]; then
+ if ps -p ${KYLIN_PID} > /dev/null; then
+ echo "Kylin is running, will be killed. (pid=${KYILN_PID})"
+ kill -9 ${KYLIN_PID}
+ fi
+fi
+
+rm -rf ${TARGET_PATH}/*kylin*
+tar -zxvf $PKG_PATH -C ${TARGET_PATH}
+
+cd ${TARGET_PATH}/*kylin*/
+export KYLIN_HOME=`pwd`
+cd -
+
+${KYLIN_HOME}/bin/metastore.sh reset
+
+# Test stage
+${KYLIN_HOME}/bin/sample.sh
+${KYLIN_HOME}/bin/kylin.sh start
+
+echo "Wait 3 minutes for service start."
+sleep 3m
+
+cd smoke-test
+python testBuildCube.py || { exit 1; }
+python testQuery.py || { exit 1; }
+cd ..
+
+# Tear down stage
+${KYLIN_HOME}/bin/metastore.sh clean --delete true
+${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.storage.hbase.util.StorageCleanupJob --delete true
+${KYLIN_HOME}/bin/metastore.sh reset
+${KYLIN_HOME}/bin/kylin.sh stop
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/1411f890/build/smoke-test/sql/sql1.json
----------------------------------------------------------------------
diff --git a/build/smoke-test/sql/sql1.json b/build/smoke-test/sql/sql1.json
new file mode 100644
index 0000000..b0b7c77
--- /dev/null
+++ b/build/smoke-test/sql/sql1.json
@@ -0,0 +1,38 @@
+{
+ "cube": "kylin_sales_cube",
+ "partial": false,
+ "affectedRowCount": 0,
+ "isException": false,
+ "storageCacheUsed": false,
+ "results": [
+ [
+ "10000"
+ ]
+ ],
+ "hitExceptionCache": false,
+ "exceptionMessage": null,
+ "totalScanCount": 1,
+ "columnMetas": [
+ {
+ "scale": 0,
+ "columnType": -5,
+ "name": "EXPR$0",
+ "searchable": false,
+ "currency": false,
+ "autoIncrement": false,
+ "definitelyWritable": false,
+ "tableName": null,
+ "precision": 19,
+ "displaySize": 19,
+ "label": "EXPR$0",
+ "writable": false,
+ "caseSensitive": true,
+ "readOnly": true,
+ "catelogName": null,
+ "signed": true,
+ "columnTypeName": "BIGINT",
+ "schemaName": null,
+ "isNullable": 0
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/1411f890/build/smoke-test/sql/sql1.sql
----------------------------------------------------------------------
diff --git a/build/smoke-test/sql/sql1.sql b/build/smoke-test/sql/sql1.sql
new file mode 100644
index 0000000..ff13294
--- /dev/null
+++ b/build/smoke-test/sql/sql1.sql
@@ -0,0 +1 @@
+select count(*) from kylin_sales
http://git-wip-us.apache.org/repos/asf/kylin/blob/1411f890/build/smoke-test/testBuildCube.py
----------------------------------------------------------------------
diff --git a/build/smoke-test/testBuildCube.py b/build/smoke-test/testBuildCube.py
new file mode 100644
index 0000000..693d678
--- /dev/null
+++ b/build/smoke-test/testBuildCube.py
@@ -0,0 +1,94 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is python unittest used in smoke-test.sh, aim to testing building cubes via rest APIs.
+
+import unittest
+import requests
+import json
+import time
+
+
+class testBuildCube(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+ def testBuild(self):
+ base_url = "http://sandbox:7070/kylin/api"
+ url = base_url + "/cubes/kylin_sales_cube/rebuild"
+ headers = {
+ 'content-type': "application/json",
+ 'authorization': "Basic QURNSU46S1lMSU4=",
+ 'cache-control': "no-cache"
+ }
+
+ # reload metadata before build cubes
+ cache_response = requests.request("PUT", base_url + "/cache/all/all/update", headers=headers)
+ self.assertEqual(cache_response.status_code, 200, 'Metadata cache not refreshed.')
+
+ payload = "{\"startTime\": 1325376000000, \"endTime\": 1456790400000, \"buildType\":\"BUILD\"}"
+ status_code = 0
+ try_time = 1
+ while status_code != 200 and try_time <= 3:
+ print 'Submit build job, try_time = ' + str(try_time)
+ try:
+ response = requests.request("PUT", url, data=payload, headers=headers)
+ status_code = response.status_code
+ except:
+ status_code = 0
+ pass
+ if status_code != 200:
+ time.sleep(60)
+ try_time += 1
+
+ self.assertEqual(status_code, 200, 'Build job submitted failed.')
+
+ if status_code == 200:
+ print 'Build job is submitted...'
+ job_response = json.loads(response.text)
+ job_uuid = job_response['uuid']
+ job_url = base_url + "/jobs/" + job_uuid
+ job_response = requests.request("GET", job_url, headers=headers)
+
+ self.assertEqual(job_response.status_code, 200, 'Build job information fetched failed.')
+
+ job_info = json.loads(job_response.text)
+ job_status = job_info['job_status']
+ try_time = 1
+ while job_status in ('RUNNING', 'PENDING') and try_time <= 20:
+ print 'Wait for job complete, try_time = ' + str(try_time)
+ try:
+ job_response = requests.request("GET", job_url, headers=headers)
+ job_info = json.loads(job_response.text)
+ job_status = job_info['job_status']
+ except:
+ job_status = 'UNKNOWN'
+ pass
+ if job_status in ('RUNNING', 'PENDING', 'UNKNOWN'):
+ time.sleep(60)
+ try_time += 1
+
+ self.assertEquals(job_status, 'FINISHED', 'Build cube failed, job status is ' + job_status)
+ print 'Job complete.'
+
+
+if __name__ == '__main__':
+ print 'Test Build Cube for Kylin sample.'
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/kylin/blob/1411f890/build/smoke-test/testQuery.py
----------------------------------------------------------------------
diff --git a/build/smoke-test/testQuery.py b/build/smoke-test/testQuery.py
new file mode 100644
index 0000000..3772333
--- /dev/null
+++ b/build/smoke-test/testQuery.py
@@ -0,0 +1,62 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is python unittest used in smoke-test.sh, aim to testing query via rest APIs.
+
+import unittest
+import requests
+import json
+import glob
+
+
+class testQuery(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+ def testBuild(self):
+ base_url = "http://sandbox:7070/kylin/api"
+ url = base_url + "/query"
+ headers = {
+ 'content-type': "application/json",
+ 'authorization': "Basic QURNSU46S1lMSU4=",
+ 'cache-control': "no-cache"
+ }
+
+ sql_files = glob.glob('sql/*.sql')
+ index = 0
+ for sql_file in sql_files:
+ index += 1
+ sql_statement = open(sql_file).read().strip()
+ payload = "{\"sql\": \"" + sql_statement + "\", \"offset\": 0, \"limit\": \"50000\", \"acceptPartial\":false, \"project\":\"learn_kylin\"}"
+ print 'Test Query #' + str(index) + ': \n' + sql_statement
+ response = requests.request("POST", url, data=payload, headers=headers)
+
+ self.assertEqual(response.status_code, 200, 'Query failed.')
+ actual_result = json.loads(response.text)
+ print 'Query duration: ' + str(actual_result['duration']) + 'ms'
+ del actual_result['duration']
+
+ expect_result = json.loads(open(sql_file[:-4] + '.json').read().strip())
+ self.assertEqual(actual_result, expect_result, 'Query result does not equal.')
+
+
+if __name__ == '__main__':
+ print 'Test Query for Kylin sample.'
+ unittest.main()