You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/11/08 08:13:13 UTC
[kylin] 13/13: KYLIN-4775 Minor fix
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 15866d18133ff29a70db01027f7ab4fd0ced0b70
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Thu Nov 5 16:48:52 2020 +0800
KYLIN-4775 Minor fix
---
.../kylin-system-testing/features/specs/query/query.spec | 16 ----------------
.../features/step_impl/query/query.py | 7 +++++--
build/CI/kylin-system-testing/kylin_utils/equals.py | 3 ++-
.../query/sql_result/sql_test/sql1.json | 8 +++++---
build/CI/run-ci.sh | 5 +++--
docker/docker-compose/write/write-hadoop.env | 1 +
docker/dockerfile/cluster/client/Dockerfile | 2 ++
pom.xml | 1 +
.../java/org/apache/kylin/rest/response/SQLResponse.java | 10 ++++++++++
.../java/org/apache/kylin/rest/service/QueryService.java | 12 ++++++++++++
.../org/apache/kylin/rest/response/SQLResponseTest.java | 2 +-
11 files changed, 42 insertions(+), 25 deletions(-)
diff --git a/build/CI/kylin-system-testing/features/specs/query/query.spec b/build/CI/kylin-system-testing/features/specs/query/query.spec
index 8cd3a6f..0bb72bb 100644
--- a/build/CI/kylin-system-testing/features/specs/query/query.spec
+++ b/build/CI/kylin-system-testing/features/specs/query/query.spec
@@ -1,19 +1,3 @@
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
# Kylin SQL test
Tags:4.x
diff --git a/build/CI/kylin-system-testing/features/step_impl/query/query.py b/build/CI/kylin-system-testing/features/step_impl/query/query.py
index 55f5597..32728b2 100644
--- a/build/CI/kylin-system-testing/features/step_impl/query/query.py
+++ b/build/CI/kylin-system-testing/features/step_impl/query/query.py
@@ -32,8 +32,11 @@ def query_sql_file_and_compare(sql_directory, project_name, sql_result_directory
sql = sql_file.read()
client = util.setup_instance('kylin_instance.yml')
- with open(sql_result_directory + sql_file_name.split(".")[0] + '.json', 'r', encoding='utf8') as expected_result_file:
- expected_result = json.loads(expected_result_file.read())
+ expected_result_file_name = sql_result_directory + sql_file_name.split(".")[0]
+ expected_result = None
+ if os.path.exists(expected_result_file_name):
+ with open(sql_result_directory + sql_file_name.split(".")[0] + '.json', 'r', encoding='utf8') as expected_result_file:
+ expected_result = json.loads(expected_result_file.read())
equals.compare_sql_result(sql=sql, project=project_name, kylin_client=client, expected_result=expected_result)
diff --git a/build/CI/kylin-system-testing/kylin_utils/equals.py b/build/CI/kylin-system-testing/kylin_utils/equals.py
index 6c990d4..fbb1388 100644
--- a/build/CI/kylin-system-testing/kylin_utils/equals.py
+++ b/build/CI/kylin-system-testing/kylin_utils/equals.py
@@ -221,7 +221,8 @@ def compare_sql_result(sql, project, kylin_client, cube=None, expected_result=No
assert query_result_equals(kylin_resp, pushdown_resp)
if expected_result is not None:
- print(kylin_resp.get("totalScanCount"))
+ assert expected_result.get("cube") == kylin_resp.get("cube")
+ assert expected_result.get("cuboidIds") == kylin_resp.get("cuboidIds")
assert expected_result.get("totalScanCount") == kylin_resp.get("totalScanCount")
assert expected_result.get("totalScanBytes") == kylin_resp.get("totalScanBytes")
assert expected_result.get("totalScanFiles") == kylin_resp.get("totalScanFiles")
diff --git a/build/CI/kylin-system-testing/query/sql_result/sql_test/sql1.json b/build/CI/kylin-system-testing/query/sql_result/sql_test/sql1.json
index 3c2ec22..42166d2 100644
--- a/build/CI/kylin-system-testing/query/sql_result/sql_test/sql1.json
+++ b/build/CI/kylin-system-testing/query/sql_result/sql_test/sql1.json
@@ -1,6 +1,8 @@
{
- "totalScanCount":7349,
- "totalScanBytes":229078,
- "totalScanFiles":2,
+ "cube": "CUBE[name=generic_test_cube]",
+ "cuboidIds": "20480",
+ "totalScanCount": 7349,
+ "totalScanBytes": 229078,
+ "totalScanFiles": 2,
"pushDown": false
}
\ No newline at end of file
diff --git a/build/CI/run-ci.sh b/build/CI/run-ci.sh
index acbb2c7..d4c0122 100644
--- a/build/CI/run-ci.sh
+++ b/build/CI/run-ci.sh
@@ -58,6 +58,7 @@ cat > kylin-all/conf/kylin.properties <<EOL
kylin.metadata.url=kylin_metadata@jdbc,url=jdbc:mysql://metastore-db:3306/metastore,username=kylin,password=kylin,maxActive=10,maxIdle=10
kylin.env.zookeeper-connect-string=write-zookeeper:2181
kylin.job.scheduler.default=100
+kylin.engine.spark-conf.spark.shuffle.service.enabled=false
kylin.query.pushdown.runner-class-name=org.apache.kylin.query.pushdown.PushDownRunnerSparkImpl
EOL
@@ -87,7 +88,7 @@ then
bash stop_cluster.sh
bash setup_hadoop_cluster.sh --cluster_mode write --hadoop_version 2.8.5 --hive_version 1.2.2 \
- --enable_hbase yes --hbase_version 1.1.2 --enable_ldap nosh setup_cluster.sh \
+ --enable_hbase no --hbase_version 1.1.2 --enable_ldap nosh setup_cluster.sh \
--cluster_mode write --hadoop_version 2.8.5 --hive_version 1.2.2 --enable_hbase yes \
--hbase_version 1.1.2 --enable_ldap no
cd ..
@@ -104,7 +105,7 @@ echo "Restart Kylin cluster."
cd docker
bash setup_service.sh --cluster_mode write --hadoop_version 2.8.5 --hive_version 1.2.2 \
- --enable_hbase yes --hbase_version 1.1.2 --enable_ldap nosh setup_cluster.sh \
+ --enable_hbase no --hbase_version 1.1.2 --enable_ldap nosh setup_cluster.sh \
--cluster_mode write --hadoop_version 2.8.5 --hive_version 1.2.2 --enable_hbase yes \
--hbase_version 1.1.2 --enable_ldap no
docker ps
diff --git a/docker/docker-compose/write/write-hadoop.env b/docker/docker-compose/write/write-hadoop.env
index a99c096..60588aa 100644
--- a/docker/docker-compose/write/write-hadoop.env
+++ b/docker/docker-compose/write/write-hadoop.env
@@ -32,6 +32,7 @@ YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___pe
YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle
YARN_CONF_mapreduce_jobhistory_address=write-historyserver:10020
+YARN_CONF_yarn_nodemanager_vmem___pmem___ratio=2.5
MAPRED_CONF_mapreduce_framework_name=yarn
MAPRED_CONF_mapred_child_java_opts=-Xmx4096m
diff --git a/docker/dockerfile/cluster/client/Dockerfile b/docker/dockerfile/cluster/client/Dockerfile
index 43c935e..632516e 100644
--- a/docker/dockerfile/cluster/client/Dockerfile
+++ b/docker/dockerfile/cluster/client/Dockerfile
@@ -128,6 +128,8 @@ ENV SPARK_CONF_DIR=/opt/spark-$SPARK_VERSION-bin-hadoop${SPARK_HADOOP_VERSION}/c
RUN curl -fSL "${SPARK_URL}" -o /tmp/spark.tar.gz \
&& tar -zxvf /tmp/spark.tar.gz -C /opt/ \
&& rm -f /tmp/spark.tar.gz \
+ && rm -f $SPARK_HOME/conf/hive-site.xml \
+ && ln -s $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf/hive-site.xml \
&& cp $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf \
&& cp $SPARK_HOME/yarn/*.jar $HADOOP_HOME/share/hadoop/yarn/lib
diff --git a/pom.xml b/pom.xml
index 4843dbf..9a3285d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1903,6 +1903,7 @@
<exclude>**/*.json</exclude>
<exclude>**/*.json.bad</exclude>
<exclude>**/*.md</exclude>
+ <exclude>**/*.spec</exclude>
<!-- binary files -->
<exclude>**/*.dict</exclude>
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index 35aef1c..d5d57ed 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -46,6 +46,8 @@ public class SQLResponse implements Serializable {
*/
protected String cube;
+ protected String cuboidIds;
+
// if not select query, only return affected row count
protected int affectedRowCount;
@@ -135,6 +137,14 @@ public class SQLResponse implements Serializable {
this.cube = cube;
}
+ public String getCuboidIds() {
+ return cuboidIds;
+ }
+
+ public void setCuboidIds(String cuboidIds) {
+ this.cuboidIds = cuboidIds;
+ }
+
public int getAffectedRowCount() {
return affectedRowCount;
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0ddf4db..d9fc732 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -1160,6 +1160,7 @@ public class QueryService extends BasicService {
List<String> realizations = Lists.newLinkedList();
StringBuilder cubeSb = new StringBuilder();
+ StringBuilder cuboidIdsSb = new StringBuilder();
StringBuilder logSb = new StringBuilder("Processed rows for each storageContext: ");
QueryContext queryContext = QueryContextFacade.current();
if (OLAPContext.getThreadLocalContexts() != null) { // contexts can be null in case of 'explain plan for'
@@ -1171,6 +1172,14 @@ public class QueryService extends BasicService {
if (cubeSb.length() > 0) {
cubeSb.append(",");
}
+ Cuboid cuboid = ctx.storageContext.getCuboid();
+ if (cuboid != null) {
+ //Some queries do not involve cuboid, e.g. lookup table query
+ if(cuboidIdsSb.length() >0) {
+ cuboidIdsSb.append(",");
+ }
+ cuboidIdsSb.append(cuboid.getId());
+ }
cubeSb.append(ctx.realization.getCanonicalName());
logSb.append(ctx.storageContext.getProcessedRowCount()).append(" ");
@@ -1181,11 +1190,14 @@ public class QueryService extends BasicService {
}
queryContext.setContextRealization(ctx.id, realizationName, realizationType);
}
+
+
}
logger.info(logSb.toString());
SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, isException,
exceptionMessage, isPartialResult, isPushDown);
+ response.setCuboidIds(cuboidIdsSb.toString());
response.setTotalScanCount(queryContext.getScannedRows());
response.setTotalScanFiles((queryContext.getScanFiles() < 0) ? -1 :
queryContext.getScanFiles());
diff --git a/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java b/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
index 48f4791..f1c704e 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
@@ -32,7 +32,7 @@ public class SQLResponseTest {
@Test
public void testInterfaceConsistency() throws IOException {
- String[] attrArray = new String[] { "columnMetas", "results", "cube", "affectedRowCount", "isException",
+ String[] attrArray = new String[] { "columnMetas", "results", "cube", "cuboidIds", "affectedRowCount", "isException",
"exceptionMessage", "duration", "partial", "totalScanCount", "hitExceptionCache", "storageCacheUsed",
"sparkPool", "pushDown", "traceUrl", "totalScanBytes", "totalScanFiles",
"metadataTime", "totalSparkScanTime" };