You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/12/05 09:37:19 UTC
(kylin) branch kylin5-docker-debug updated (5894eb4fa0 -> 7ca4b1a06f)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin5-docker-debug
in repository https://gitbox.apache.org/repos/asf/kylin.git
omit 5894eb4fa0 KYLIN-5730 Query dry-run for better modeling
new 7ca4b1a06f KYLIN-5730 Query dry-run for better modeling
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (5894eb4fa0)
\
N -- N -- N refs/heads/kylin5-docker-debug (7ca4b1a06f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
kystudio/src/components/query/insight.vue | 2 +-
kystudio/src/components/query/query_tab.vue | 4 +++-
.../test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java | 1 +
.../org/apache/kylin/rest/service/QueryMetricsListenerTest.java | 3 +--
.../src/main/java/org/apache/kylin/query/engine/QueryExec.java | 6 +++---
5 files changed, 9 insertions(+), 7 deletions(-)
(kylin) 01/01: KYLIN-5730 Query dry-run for better modeling
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5-docker-debug
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7ca4b1a06faa78e1ae530a07254391e23e624997
Author: XiaoxiangYu <xx...@apache.org>
AuthorDate: Tue Dec 5 12:04:26 2023 +0800
KYLIN-5730 Query dry-run for better modeling
---
build/conf/kylin-server-log4j.xml | 2 +-
.../contributor/sandbox-standalone/README.md | 1 +
.../sandbox-standalone/conf/capacity-scheduler.xml | 122 +++
.../sandbox-standalone/conf/core-site.xml | 29 +
.../sandbox-standalone/conf/fairscheduler.xml | 47 +
.../sandbox-standalone/conf/hdfs-site.xml | 36 +
.../sandbox-standalone/conf/hive-site.xml | 44 +
.../conf/kylin.properties | 28 +-
.../sandbox-standalone/conf/mapred-site.xml | 20 +
.../sandbox-standalone/conf/yarn-site.xml | 75 ++
dev-support/contributor/{ => sandbox}/README.md | 0
.../contributor/sandbox/conf/kylin.properties | 10 +-
.../release-machine/Dockerfile_2 | 9 +-
.../release-machine/README.md | 34 +-
.../release-machine/{ => bin}/entrypoint.sh | 16 +-
.../release-machine/{ => bin}/release-publish.sh | 298 ++++---
.../release-machine/create-release-machine.sh | 2 +-
.../release-machine/run-release-machine.sh | 56 ++
.../release-manager/release-process-definition.md | 21 +-
.../standalone-docker/all_in_one/Dockerfile_kylin | 2 +-
.../standalone-docker/all_in_one/README.md | 297 +++++++
.../{build_image.sh => build_and_run.sh} | 30 +-
.../conf/hadoop-dev/capacity-scheduler.xml | 134 +++
.../all_in_one/conf/hadoop-dev/core-site.xml | 29 +
.../all_in_one/conf/hadoop-dev/hdfs-site.xml | 36 +
.../all_in_one/conf/hadoop-dev/mapred-site.xml | 20 +
.../all_in_one/conf/hadoop-dev/yarn-site.xml | 65 ++
.../{kylin => kylin-dev}/kylin.properties.override | 32 +-
.../conf/kylin/kylin.properties.override | 30 +-
.../standalone-docker/all_in_one/conf/mysql/my.cnf | 4 +
.../Dockerfile_kylin_dev} | 26 +-
.../build_and_run.sh} | 62 +-
.../entrypoint-dev.sh} | 48 +-
.../all_in_one/{ => scripts}/entrypoint.sh | 0
.../release-manager/tpch-benchmark/README.md | 990 +++++++++++++++++++++
.../tpch-benchmark/resources/TPCH-Schema.png | Bin 0 -> 316488 bytes
.../tpch-benchmark/scripts/create-all-tables.sql | 215 +++++
.../scripts/transform-to-parquet.scala | 145 +++
.../scripts/update-hive-column-comment.sql | 98 ++
.../components/layout/layout_left_right_top.vue | 3 +-
kystudio/src/components/query/insight.vue | 2 +-
kystudio/src/components/query/query_tab.vue | 6 +-
.../setting/SettingBasic/SettingBasic.vue | 4 +-
pom.xml | 2 +-
.../apache/kylin/rest/config/MetricsConfig.java | 25 +-
.../rest/config/initialize/MetricsRegistry.java | 10 +-
.../config/initialize/QueryMetricsListener.java | 9 +-
.../java/org/apache/kylin/common/KapConfig.java | 1 -
.../org/apache/kylin/common/KylinConfigBase.java | 5 +-
.../java/org/apache/kylin/common/KylinVersion.java | 6 +-
.../java/org/apache/kylin/common/QueryContext.java | 21 +
.../common/exception/DryRunSucceedException.java | 36 +
.../metadata/cube/cuboid/NAggregationGroup.java | 3 -
.../kylin/metadata/cube/model/IndexPlan.java | 1 +
.../kylin/metadata/cube/model/NDataflow.java | 2 +-
.../metadata/query/NativeQueryRealization.java | 2 +
.../kylin/metadata/realization/SQLDigest.java | 2 +-
.../apache/kylin/common/metrics/MetricsGroup.java | 4 +
.../kylin/common/metric/InfluxMetricsTest.java | 1 +
.../apache/kylin/query/relnode/ContextUtil.java | 4 +
.../apache/kylin/query/relnode/OLAPContext.java | 39 +
.../apache/kylin/query/routing/QueryRouter.java | 28 +
.../kylin/query/routing/RealizationChooser.java | 42 +-
.../org/apache/kylin/query/util/QueryUtil.java | 2 +-
.../rest/controller/v2/NQueryMetaController.java | 2 -
.../rest/service/QueryMetricsListenerTest.java | 1 +
.../org/apache/kylin/query/engine/QueryExec.java | 83 +-
.../apache/kylin/query/engine/SQLConverter.java | 2 +-
.../kylin/query/util/QueryContextCutter.java | 6 +-
.../kylin/query/util/QueryModelPriorities.java | 2 -
src/server/src/main/resources/log4j2.xml | 1 +
.../apache/kylin/query/runtime/SparkEngine.java | 5 +
72 files changed, 3201 insertions(+), 274 deletions(-)
diff --git a/build/conf/kylin-server-log4j.xml b/build/conf/kylin-server-log4j.xml
index 14d22263ff..711cac2cad 100644
--- a/build/conf/kylin-server-log4j.xml
+++ b/build/conf/kylin-server-log4j.xml
@@ -101,7 +101,7 @@
<Logger name="org.apache.spark.ContextCleaner" level="WARN"/>
<Logger name="org.apache.kylin.ext" level="INFO"/>
<!-- Query log -->
- <Logger name="org.apache.kylin.query" level="INFO"/>
+ <Logger name="org.apache.kylin.query" level="DEBUG"/>
<Logger name="NDataflowCapabilityChecker" level="INFO" />
<Logger name="org.apache.kylin.common.util.CheckUtil" level="INFO" />
<Logger name="NQueryLayoutChooser" level="INFO" />
diff --git a/dev-support/contributor/sandbox-standalone/README.md b/dev-support/contributor/sandbox-standalone/README.md
new file mode 100644
index 0000000000..f3dbcae5cc
--- /dev/null
+++ b/dev-support/contributor/sandbox-standalone/README.md
@@ -0,0 +1 @@
+## How to debug Kylin with docker
diff --git a/dev-support/contributor/sandbox-standalone/conf/capacity-scheduler.xml b/dev-support/contributor/sandbox-standalone/conf/capacity-scheduler.xml
new file mode 100644
index 0000000000..ff424705b9
--- /dev/null
+++ b/dev-support/contributor/sandbox-standalone/conf/capacity-scheduler.xml
@@ -0,0 +1,122 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-applications</name>
+ <value>4</value>
+ <description>
+ Maximum number of applications that can be pending and running.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.5</value>
+ <description>
+ Maximum percent of resources in the cluster which can be used to run
+ application masters i.e. controls number of concurrent running
+ applications.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.resource-calculator</name>
+ <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+ <description>
+ The ResourceCalculator implementation to be used to compare
+ Resources in the scheduler.
+ The default i.e. DefaultResourceCalculator only uses Memory while
+ DominantResourceCalculator uses dominant-resource to compare
+ multi-dimensional resources such as Memory, CPU etc.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ <value>default</value>
+ <description>
+ The queues at the this level (root is the root queue).
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.capacity</name>
+ <value>100</value>
+ <description>Default queue target capacity.</description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
+ <value>1</value>
+ <description>
+ Default queue user limit a percentage from 0.0 to 1.0.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
+ <value>100</value>
+ <description>
+ The maximum capacity of the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.state</name>
+ <value>RUNNING</value>
+ <description>
+ The state of the default queue. State can be one of RUNNING or STOPPED.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
+ <value>*</value>
+ <description>
+ The ACL of who can submit jobs to the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
+ <value>*</value>
+ <description>
+ The ACL of who can administer jobs on the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.node-locality-delay</name>
+ <value>40</value>
+ <description>
+ Number of missed scheduling opportunities after which the CapacityScheduler
+ attempts to schedule rack-local containers.
+ Typically this should be set to number of nodes in the cluster, By default is setting
+ approximately number of nodes in one rack which is 40.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.queue-mappings-override.enable</name>
+ <value>false</value>
+ <description>
+ If a queue mapping is present, will it override the value specified
+ by the user? This can be used by administrators to place jobs in queues
+ that are different than the one specified by the user.
+ The default is false.
+ </description>
+ </property>
+
+</configuration>
diff --git a/dev-support/contributor/sandbox-standalone/conf/core-site.xml b/dev-support/contributor/sandbox-standalone/conf/core-site.xml
new file mode 100644
index 0000000000..2e7e4a5671
--- /dev/null
+++ b/dev-support/contributor/sandbox-standalone/conf/core-site.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>hadoop.tmp.dir</name>
+ <value>/data/hadoop</value>
+ <description>Abase for other temporary directories.</description>
+ </property>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://kylin5-machine:9000</value>
+ </property>
+</configuration>
diff --git a/dev-support/contributor/sandbox-standalone/conf/fairscheduler.xml b/dev-support/contributor/sandbox-standalone/conf/fairscheduler.xml
new file mode 100644
index 0000000000..0e754f6e11
--- /dev/null
+++ b/dev-support/contributor/sandbox-standalone/conf/fairscheduler.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<allocations>
+ <pool name="query_pushdown">
+ <schedulingMode>FAIR</schedulingMode>
+ <weight>1</weight>
+ <minShare>1</minShare>
+ </pool>
+ <pool name="extreme_heavy_tasks">
+ <schedulingMode>FAIR</schedulingMode>
+ <weight>3</weight>
+ <minShare>1</minShare>
+ </pool>
+ <pool name="heavy_tasks">
+ <schedulingMode>FAIR</schedulingMode>
+ <weight>5</weight>
+ <minShare>1</minShare>
+ </pool>
+ <pool name="lightweight_tasks">
+ <schedulingMode>FAIR</schedulingMode>
+ <weight>10</weight>
+ <minShare>1</minShare>
+ </pool>
+ <pool name="vip_tasks">
+ <schedulingMode>FAIR</schedulingMode>
+ <weight>15</weight>
+ <minShare>1</minShare>
+ </pool>
+</allocations>
+
diff --git a/dev-support/contributor/sandbox-standalone/conf/hdfs-site.xml b/dev-support/contributor/sandbox-standalone/conf/hdfs-site.xml
new file mode 100644
index 0000000000..cd4edeedd3
--- /dev/null
+++ b/dev-support/contributor/sandbox-standalone/conf/hdfs-site.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>dfs.datanode.address</name>
+ <value>kylin5-machine:9866</value>
+ </property>
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>/data/hadoop/dfs/name</value>
+ </property>
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/data/hadoop/dfs/data</value>
+ </property>
+</configuration>
diff --git a/dev-support/contributor/sandbox-standalone/conf/hive-site.xml b/dev-support/contributor/sandbox-standalone/conf/hive-site.xml
new file mode 100644
index 0000000000..a33245d90c
--- /dev/null
+++ b/dev-support/contributor/sandbox-standalone/conf/hive-site.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>hive.metastore.db.type</name>
+ <value>mysql</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:mysql://kylin5-machine:3306/hive?createDatabaseIfNotExist=true</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>root</value>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>123456</value>
+ </property>
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ </property>
+</configuration>
diff --git a/dev-support/contributor/sandbox/conf/kylin.properties b/dev-support/contributor/sandbox-standalone/conf/kylin.properties
similarity index 89%
copy from dev-support/contributor/sandbox/conf/kylin.properties
copy to dev-support/contributor/sandbox-standalone/conf/kylin.properties
index b172f77106..4a05b02c0f 100644
--- a/dev-support/contributor/sandbox/conf/kylin.properties
+++ b/dev-support/contributor/sandbox-standalone/conf/kylin.properties
@@ -15,6 +15,8 @@
# limitations under the License.
#
+
+kylin.query.timeout-seconds=3000
### METADATA | ENV ###
# Env DEV|QA|PROD
kylin.env=DEV
@@ -27,12 +29,14 @@ spring.session.timeout=10800
kylin.metadata.random-admin-password.enabled=false
kylin.query.cache-enabled=false
-kylin.server.leader-race.heart-beat-interval=240
-kylin.server.leader-race.heart-beat-timeout=360
+
+kylin.server.leader-race.enabled=false
+kylin.server.leader-race.heart-beat-interval=2400
+kylin.server.leader-race.heart-beat-timeout=3600
kylin.task.jstack-dump-enabled=false
# The metadata store, by default stored in jdbc
-kylin.metadata.url=kylin_metadata@jdbc,driverClassName=com.mysql.jdbc.Driver,url=jdbc:mysql://mysql:3306/kylin?useSSL=false,createDatabaseIfNotExist=true,username=root,password=root
+kylin.metadata.url=kylin_dev@jdbc,driverClassName=com.mysql.jdbc.Driver,url=jdbc:mysql://kylin5-machine:3306/kylin?useSSL=false,createDatabaseIfNotExist=true,username=root,password=123456
kylin.source.record-source-usage-enabled=false
@@ -46,9 +50,9 @@ kylin.server.mode=all
### STORAGE ###
# Working folder in HDFS, make sure user has the right access to the hdfs directory
-kylin.env.hdfs-working-dir=/kylin
+kylin.env.hdfs-working-dir=hdfs://kylin5-machine:9000/kylin
-kylin.env.zookeeper-connect-string=zookeeper:2181
+kylin.env.zookeeper-connect-string=kylin5-machine:2181
### JOB ###
@@ -80,16 +84,16 @@ kylin.job.max-concurrent-jobs=3
kylin.engine.spark-conf.spark.sql.hive.metastore.version=1.2.2
-kylin.engine.spark-conf.spark.sql.hive.metastore.jars=${KYLIN_HOME}/build/spark/hive_1_2_2/*
+kylin.engine.spark-conf.spark.sql.hive.metastore.jars=${SPARK_HOME}/hive_1_2_2/*
kylin.engine.spark-conf.spark.driver.memory=512m
-kylin.engine.spark-conf.spark.executor.memory=4G
+kylin.engine.spark-conf.spark.executor.memory=1G
kylin.engine.spark-conf.spark.executor.memoryOverhead=1G
kylin.engine.spark-conf.spark.executor.cores=1
kylin.engine.spark-conf.spark.executor.instances=1
kylin.engine.spark-conf.spark.sql.autoBroadcastJoinThreshold=-1
-kylin.engine.spark.job-jar=../assembly/target/kylin-assembly-5.0.0-SNAPSHOT-job.jar
+kylin.engine.spark.job-jar=${KYLIN_HOME}/src/assembly/target/kylin-assembly-5.0.0-SNAPSHOT-job.jar
@@ -171,13 +175,13 @@ kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=
kylin.storage.columnar.spark-conf.spark.yarn.am.memory=512m
kylin.storage.columnar.spark-conf.spark.executor.cores=1
kylin.storage.columnar.spark-conf.spark.executor.instances=1
-kylin.storage.columnar.spark-conf.spark.eventLog.enabled=true
+kylin.storage.columnar.spark-conf.spark.eventLog.enabled=false
kylin.storage.columnar.spark-conf.spark.history.fs.logDirectory=${kylin.env.hdfs-working-dir}/sparder-history
kylin.storage.columnar.spark-conf.spark.eventLog.dir=${kylin.env.hdfs-working-dir}/sparder-history
kylin.storage.columnar.spark-conf.spark.eventLog.rolling.enabled=true
kylin.storage.columnar.spark-conf.spark.eventLog.rolling.maxFileSize=50m
kylin.storage.columnar.spark-conf.spark.sql.hive.metastore.version=1.2.2
-kylin.storage.columnar.spark-conf.spark.sql.hive.metastore.jars=${KYLIN_HOME}/build/spark/hive_1_2_2/*
+kylin.storage.columnar.spark-conf.spark.sql.hive.metastore.jars=${SPARK_HOME}/hive_1_2_2/*
# ==================== ASYNC QUERY ====================
# sandbox mode should use smaller resources
@@ -196,7 +200,7 @@ kylin.query.async-query.spark-conf.spark.eventLog.dir=${kylin.env.hdfs-working-d
kylin.query.async-query.spark-conf.spark.eventLog.rolling.enabled=true
kylin.query.async-query.spark-conf.spark.eventLog.rolling.maxFileSize=50m
kylin.query.async-query.spark-conf.spark.sql.hive.metastore.version=1.2.2
-kylin.query.async-query.spark-conf.spark.sql.hive.metastore.jars=${KYLIN_HOME}/build/spark/hive_1_2_2/*
+kylin.query.async-query.spark-conf.spark.sql.hive.metastore.jars=${SPARK_HOME}/hive_1_2_2/*
kylin.storage.columnar.spark-conf.spark.sql.parquet.columnarReaderBatchSize=1024
@@ -204,7 +208,7 @@ kylin.storage.columnar.spark-conf.spark.sql.parquet.columnarReaderBatchSize=1024
kylin.smart.conf.propose-runner-type=in-memory
#streaming
-kylin.streaming.spark-conf.spark.sql.hive.metastore.jars=${KYLIN_HOME}/build/spark/hive_1_2_2/*
+kylin.streaming.spark-conf.spark.sql.hive.metastore.jars=${SPARK_HOME}/hive_1_2_2/*
kylin.streaming.spark.job-jar=../assembly/target/kylin-assembly-5.0.0-SNAPSHOT-job.jar
kylin.query.escape-default-keyword=true
diff --git a/dev-support/contributor/sandbox-standalone/conf/mapred-site.xml b/dev-support/contributor/sandbox-standalone/conf/mapred-site.xml
new file mode 100644
index 0000000000..ac8ef33969
--- /dev/null
+++ b/dev-support/contributor/sandbox-standalone/conf/mapred-site.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/dev-support/contributor/sandbox-standalone/conf/yarn-site.xml b/dev-support/contributor/sandbox-standalone/conf/yarn-site.xml
new file mode 100644
index 0000000000..d3d834cab5
--- /dev/null
+++ b/dev-support/contributor/sandbox-standalone/conf/yarn-site.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>kylin5-machine:8030</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>kylin5-machine:8032</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.hostname</name>
+ <value>kylin5-machine</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.hostname</name>
+ <value>kylin5-machine</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>6144</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>14336</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>7</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.6</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+ <value>98.5</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-address</name>
+ <value>kylin5-machine:2181</value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/dev-support/contributor/README.md b/dev-support/contributor/sandbox/README.md
similarity index 100%
rename from dev-support/contributor/README.md
rename to dev-support/contributor/sandbox/README.md
diff --git a/dev-support/contributor/sandbox/conf/kylin.properties b/dev-support/contributor/sandbox/conf/kylin.properties
index b172f77106..2438d30fc4 100644
--- a/dev-support/contributor/sandbox/conf/kylin.properties
+++ b/dev-support/contributor/sandbox/conf/kylin.properties
@@ -80,7 +80,7 @@ kylin.job.max-concurrent-jobs=3
kylin.engine.spark-conf.spark.sql.hive.metastore.version=1.2.2
-kylin.engine.spark-conf.spark.sql.hive.metastore.jars=${KYLIN_HOME}/build/spark/hive_1_2_2/*
+kylin.engine.spark-conf.spark.sql.hive.metastore.jars=${SPARK_HOME}/hive_1_2_2/*
kylin.engine.spark-conf.spark.driver.memory=512m
kylin.engine.spark-conf.spark.executor.memory=4G
@@ -89,7 +89,7 @@ kylin.engine.spark-conf.spark.executor.cores=1
kylin.engine.spark-conf.spark.executor.instances=1
kylin.engine.spark-conf.spark.sql.autoBroadcastJoinThreshold=-1
-kylin.engine.spark.job-jar=../assembly/target/kylin-assembly-5.0.0-SNAPSHOT-job.jar
+kylin.engine.spark.job-jar=${KYLIN_HOME}/src/assembly/target/kylin-assembly-5.0.0-SNAPSHOT-job.jar
@@ -177,7 +177,7 @@ kylin.storage.columnar.spark-conf.spark.eventLog.dir=${kylin.env.hdfs-working-di
kylin.storage.columnar.spark-conf.spark.eventLog.rolling.enabled=true
kylin.storage.columnar.spark-conf.spark.eventLog.rolling.maxFileSize=50m
kylin.storage.columnar.spark-conf.spark.sql.hive.metastore.version=1.2.2
-kylin.storage.columnar.spark-conf.spark.sql.hive.metastore.jars=${KYLIN_HOME}/build/spark/hive_1_2_2/*
+kylin.storage.columnar.spark-conf.spark.sql.hive.metastore.jars=${SPARK_HOME}/hive_1_2_2/*
# ==================== ASYNC QUERY ====================
# sandbox mode should use smaller resources
@@ -196,7 +196,7 @@ kylin.query.async-query.spark-conf.spark.eventLog.dir=${kylin.env.hdfs-working-d
kylin.query.async-query.spark-conf.spark.eventLog.rolling.enabled=true
kylin.query.async-query.spark-conf.spark.eventLog.rolling.maxFileSize=50m
kylin.query.async-query.spark-conf.spark.sql.hive.metastore.version=1.2.2
-kylin.query.async-query.spark-conf.spark.sql.hive.metastore.jars=${KYLIN_HOME}/build/spark/hive_1_2_2/*
+kylin.query.async-query.spark-conf.spark.sql.hive.metastore.jars=${SPARK_HOME}/hive_1_2_2/*
kylin.storage.columnar.spark-conf.spark.sql.parquet.columnarReaderBatchSize=1024
@@ -204,7 +204,7 @@ kylin.storage.columnar.spark-conf.spark.sql.parquet.columnarReaderBatchSize=1024
kylin.smart.conf.propose-runner-type=in-memory
#streaming
-kylin.streaming.spark-conf.spark.sql.hive.metastore.jars=${KYLIN_HOME}/build/spark/hive_1_2_2/*
+kylin.streaming.spark-conf.spark.sql.hive.metastore.jars=${SPARK_HOME}/hive_1_2_2/*
kylin.streaming.spark.job-jar=../assembly/target/kylin-assembly-5.0.0-SNAPSHOT-job.jar
kylin.query.escape-default-keyword=true
diff --git a/dev-support/release-manager/release-pipeline-docker/release-machine/Dockerfile_2 b/dev-support/release-manager/release-pipeline-docker/release-machine/Dockerfile_2
index da75a7c459..8c618c9941 100644
--- a/dev-support/release-manager/release-pipeline-docker/release-machine/Dockerfile_2
+++ b/dev-support/release-manager/release-pipeline-docker/release-machine/Dockerfile_2
@@ -22,6 +22,11 @@
# 2. https://infra.apache.org/release-publishing.html
FROM apachekylin/release-machine:5.0-base
+WORKDIR /root/release-manager/
COPY conf/settings.xml /root/.m2/settings.xml
-COPY release-publish.sh /root/release-publish.sh
-RUN chmod u+x /root/release-publish.sh
\ No newline at end of file
+COPY bin/release-publish.sh /root/release-publish.sh
+COPY bin/entrypoint.sh /root/entrypoint.sh
+RUN chmod u+x /root/release-publish.sh
+RUN chmod u+x /root/entrypoint.sh
+
+ENTRYPOINT ["/root/entrypoint.sh"]
\ No newline at end of file
diff --git a/dev-support/release-manager/release-pipeline-docker/release-machine/README.md b/dev-support/release-manager/release-pipeline-docker/release-machine/README.md
index 3528467f48..dbf570db8e 100644
--- a/dev-support/release-manager/release-pipeline-docker/release-machine/README.md
+++ b/dev-support/release-manager/release-pipeline-docker/release-machine/README.md
@@ -27,24 +27,32 @@ It also provided a way to publish documentation for Kylin 5.
-[ ] Update `CURRENT_KYLIN_VERSION` in `KylinVersion.java` .
```bash
+
+docker stop release-machine
+docker container rm release-machine
+bash create-release-machine.sh
+
+
WORKDIR=~/release-manager/working-dir
-rm -rf $WORKDIR
-docker start release-machine \
+# rm -rf $WORKDIR
+mkdir -p $RM_WORKDIR
+docker run --name release-machine \
+ --hostname release-machine \
+ --interactive \
--volume "$WORKDIR:/root/kylin-folder" \
--volume ~/.gnupg/:/root/.gnupg \
- --env GIT_USERNAME='???' \
- --env GPG_KEY=??? \
- --env GPG_PASSPHRASE=??? \
- --env ASF_USERNAME=??? \
- --env ASF_PASSWORD='???' \
- --env GIT_BRANCH=??? \
- --env RELEASE_VERSION=??? \
- --env NEXT_RELEASE_VERSION=??? \
- --env RC_NUMBER=??? \
+ --env GIT_USERNAME=$GIT_USERNAME \
+ --env GPG_KEY=$GPG_KEY \
+ --env GPG_PASSPHRASE=$GPG_PASSPHRASE \
+ --env ASF_USERNAME=$ASF_USERNAME \
+ --env ASF_PASSWORD=$ASF_PASSWORD \
+ --env GIT_BRANCH=kylin5\
+ --env RELEASE_VERSION=5.0.0 \
+ --env NEXT_RELEASE_VERSION=5.0.1 \
+ --env RC_NUMBER=0 \
-p 4040:4040 \
-t release-machine:latest \
-
- bash
+ bash
```
| Name | Comment |
diff --git a/dev-support/release-manager/release-pipeline-docker/release-machine/entrypoint.sh b/dev-support/release-manager/release-pipeline-docker/release-machine/bin/entrypoint.sh
similarity index 79%
rename from dev-support/release-manager/release-pipeline-docker/release-machine/entrypoint.sh
rename to dev-support/release-manager/release-pipeline-docker/release-machine/bin/entrypoint.sh
index 8dc1ab4e32..3d89218f56 100644
--- a/dev-support/release-manager/release-pipeline-docker/release-machine/entrypoint.sh
+++ b/dev-support/release-manager/release-pipeline-docker/release-machine/bin/entrypoint.sh
@@ -20,4 +20,18 @@
# */
#
-# TODO:Run release-publish.sh instead.
\ No newline at end of file
+WDR=/root/release-manager/
+mv /root/release-publish.sh $WDR
+
+source /root/.bashrc
+bash $HOME/.nvm/install.sh
+nvm list
+
+bash -x release-publish.sh \
+ -b $GIT_BRANCH \
+ -g apache \
+ reset
+while :
+do
+ sleep 20
+done
\ No newline at end of file
diff --git a/dev-support/release-manager/release-pipeline-docker/release-machine/release-publish.sh b/dev-support/release-manager/release-pipeline-docker/release-machine/bin/release-publish.sh
similarity index 56%
rename from dev-support/release-manager/release-pipeline-docker/release-machine/release-publish.sh
rename to dev-support/release-manager/release-pipeline-docker/release-machine/bin/release-publish.sh
index 1382b874b0..3476b7d9d7 100644
--- a/dev-support/release-manager/release-pipeline-docker/release-machine/release-publish.sh
+++ b/dev-support/release-manager/release-pipeline-docker/release-machine/bin/release-publish.sh
@@ -23,23 +23,30 @@
# https://stackoverflow.com/questions/57591432/gpg-signing-failed-inappropriate-ioctl-for-device-on-macos-with-maven
GPG_TTY=$(tty)
export GPG_TTY
-CUR_DATE=$(date "+%Y-%m-%d")
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
set -e
function exit_with_usage {
cat << EOF
-usage: release-publish.sh <publish-snapshot|publish-release|preview-site|...>
-Creates build deliverables from a Kylin commit.
-
-Top level targets are
- // package:
- publish-snapshot: Publish snapshot release to Apache snapshots
- publish-release: Publish a release to Apache release repo
+usage: release-publish.sh options command
+
+Options are
+ -b ,GIT_BRANCH
+ -d ,DRY_RUN, for maven-release-plugin only
+ -g ,GITHUB_UID, part of you Github URL
+ -o ,MODE_OFFICIAL, if you are using gitbox or github
+ -r ,MODE_RELEASE, if you are packaging or releasing
+ -v ,MODE_VERBOSE, if you want to double check important message
+
+Command are
+ init: Clone all source repo
reset: Clean when after a fail release attempt
+ publish-snapshot(RM only): Publish snapshot release to Apache snapshots
+ package:
+ publish-release(RM only): Publish a release to Apache release repo
preview-site: Build Kylin website on docker, so you can check/preview website in localhost:7070
- publish-site: After checked, you can upload content to apache.org
+ publish-site(RM only): After checked, you can upload content to apache.org
EOF
exit 0
}
@@ -52,7 +59,7 @@ EOF
}
function ask_confirm {
- if [ "$MODE" = "batch" ] ;then
+ if [ "$MODE_VERBOSE" = "0" ] ;then
return 0
fi
read -p "$1 Will you continue? [y/n] " ANSWER
@@ -62,25 +69,8 @@ function ask_confirm {
fi
}
-if [ $# -eq 0 ]; then
- exit_with_usage
-else
- if [ "$1" = "reset" ] || \
- [ "$1" = "publish-snapshot" ] || \
- [ "$1" = "publish-release" ] || \
- [ "$1" = "preview-site" ] || \
- [ "$1" = "publish-site" ];
- then
- ask_confirm "You are running step [$1] as release manager"
- RELEASE_STEP=$1
- else
- echo "Your input $1 is not valid."
- exit_with_usage
- fi
-fi
-
function read_config {
- if [ "$MODE" = "batch" ] ;then
+ if [ "$MODE_VERBOSE" = "0" ] ;then
echo "$2"
elif [ "$MODE" = "" ]; then
local PROMPT="$1"
@@ -118,96 +108,128 @@ function run_command {
}
function switch_node_for_packaging {
- # nvm use system
- node -v # 12.22.12
+ nvm use system
+ node -v
+ # should be 12.22.12
}
function switch_node_for_website {
- # nvm install 16.14.2
- # nvm use 16.14.2
- node -v # 16.14.2
+ nvm install 16.14.2
+ nvm use 16.14.2
+ node -v
+ # should be 16.14.2
}
-DRY_RUN=false
-
-ASF_USERNAME=$(read_config "Your apache id?" "$ASF_USERNAME")
-GIT_USERNAME=$(read_config "Your full name(used as author of git commit)?" "$GIT_USERNAME")
-ASF_PASSWORD=$(read_config "Your apache password?" "$ASF_PASSWORD")
-GIT_EMAIL=$ASF_USERNAME"@apache.org"
-GPG_KEY=$(read_config "GPG key of you(used to sign release candidate)?" "$GPG_KEY")
-GPG_PASSPHRASE=$(read_config "PASSPHRASE for your private GPG key?" "$GPG_PASSPHRASE")
-
-GIT_BRANCH=$(read_config "Git branch for release?" "$GIT_BRANCH")
-RELEASE_VERSION=$(read_config "Which version are you going to release?" "$RELEASE_VERSION")
-NEXT_RELEASE_VERSION=$(read_config "Which version is the next development version?" "$NEXT_RELEASE_VERSION")
-RC_NUMBER="rc"$(read_config "Number for release candidate?" "$RC_NUMBER")
-
-export working_dir=/root/kylin-folder
-svn_folder=$working_dir/svn
-source_code_folder=$working_dir/source/kylin
-
-packaging_folder=$source_code_folder/target/checkout
-svn_stage_folder=$svn_folder/dev
-rc_name=apache-kylin-"${RELEASE_VERSION}"-${RC_NUMBER}
-release_candidate_folder=$svn_stage_folder/$rc_name
-final_release_folder=$svn_folder/release
-
-branch_doc_1=document
-branch_doc_2=doc5.0
-document_folder=$working_dir/document
-document_folder_src=$document_folder/src
-document_folder_elder=$document_folder_src/$branch_doc_1
-document_folder_newer=$document_folder_src/$branch_doc_2
-document_folder_svn=$svn_folder/site
-
-LOG=$working_dir/$rc_name-$CUR_DATE.log
-rm -rf "$LOG"
-
-ASF_KYLIN_REPO="gitbox.apache.org/repos/asf/kylin.git"
-# GITHUB_REPO_URL=${GIT_REPO_URL:-https://github.com/apache/kylin.git}
-RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/kylin"
-RELEASE_LOCATION="https://dist.apache.org/repos/dist/release/kylin"
-WEBSITE_SVN="https://svn.apache.org/repos/asf/kylin/site"
-
-GPG_COMMAND="gpg -u $GPG_KEY --no-tty --batch --pinentry-mode loopback"
-
-if [[ -z "$ASF_PASSWORD" ]]; then
- echo 'The environment variable ASF_PASSWORD is not set. Enter the password.'
- echo
- stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo
-fi
+####################################################
+####################################################
+# Following is for configuring
-if [[ -z "$GPG_PASSPHRASE" ]]; then
- echo 'The environment variable GPG_PASSPHRASE is not set. Enter the passphrase to'
- echo 'unlock the GPG signing key that will be used to sign the release!'
- echo
- stty -echo && printf "GPG passphrase: " && read GPG_PASSPHRASE && printf '\n' && stty echo
-fi
+function configure_release {
+ export rm_dir=/root/release-manager
+ export working_dir=$rm_dir/kylin-folder
+ CUR_DATE=$(date "+%Y-%m-%d")
+
+
+ ### 0. identity
+ if [ "$MODE_OFFICIAL" = "1" ] ; then
+ echo "As Apache Committer, you using Gitbox."
+ ASF_USERNAME=$(read_config "Your apache id?" "$ASF_USERNAME")
+ ASF_PASSWORD=$(read_config "Your apache password?" "$ASF_PASSWORD")
+ GIT_EMAIL=$ASF_USERNAME"@apache.org"
+ else
+ echo "As developer, you using Github."
+ GIT_EMAIL=$(read_config "Your EMAIL?" "N/A")
+ fi
-for env in ASF_USERNAME GPG_PASSPHRASE RC_NUMBER GPG_KEY; do
- if [ -z "${!env}" ]; then
- echo "ERROR: $env must be set to run this script"
- exit_with_usage
+ if [ "$MODE_RELEASE" = "1" ] ; then
+ echo "Packaging and releasing"
+ GPG_KEY=$(read_config "GPG key of you(used to sign release candidate)?" "$GPG_KEY")
+ GPG_PASSPHRASE=$(read_config "PASSPHRASE for your private GPG key?" "$GPG_PASSPHRASE")
+ RELEASE_VERSION=$(read_config "Which version are you going to release?" "$RELEASE_VERSION")
+ NEXT_RELEASE_VERSION=$(read_config "Which version is the next development version?" "$NEXT_RELEASE_VERSION")
+ RC_NUMBER="rc"$(read_config "Number for release candidate?" "$RC_NUMBER")
+ else
+ echo "Only packaging"
+ fi
+ GIT_USERNAME=$(read_config "Your full name(used as author of git commit)?" "$GIT_USERNAME")
+ GIT_BRANCH=$(read_config "Git branch for release?" "$GIT_BRANCH")
+
+
+ ### 1. Source code folder
+ svn_folder=$working_dir/asf_svn
+ export source_code_folder=$working_dir/source/gitbox
+ export packaging_folder=$working_dir/source/github
+
+
+ ### 2. Release binary folder
+ export svn_stage_folder=$svn_folder/dev
+ export rc_name=apache-kylin-"${RELEASE_VERSION}"-${RC_NUMBER}
+ export release_candidate_folder=$svn_stage_folder/$rc_name
+ export final_release_folder=$svn_folder/release
+
+
+ ### 3. Official document folder
+ export branch_doc_1=document
+ export branch_doc_2=doc5.0
+ export document_folder=$working_dir/document
+ export document_folder_elder=$document_folder/outer
+ export document_folder_newer=$document_folder/inner
+ export document_folder_svn=$svn_folder/site
+
+
+ ### 4. Remote repository setting
+ # for apache committer only
+ export ASF_KYLIN_REPO="gitbox.apache.org/repos/asf/kylin.git"
+ # for all developer
+ export GITHUB_REPO_URL="https://github.com/${GITHUB_UID:apache}/kylin.git"
+ # for upload release candidate
+ export RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/kylin"
+ # for upload final approved release
+ export RELEASE_LOCATION="https://dist.apache.org/repos/dist/release/kylin"
+ # for upload official website
+ export WEBSITE_SVN="https://svn.apache.org/repos/asf/kylin/site"
+
+ if [ "$MODE_OFFICIAL" = "1" ] ; then
+ export FINAL_KYLIN_REPO="https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO"
+ else
+ export FINAL_KYLIN_REPO=$GITHUB_REPO_URL
fi
-done
+
+
+ ### 5. Misc
+ export GPG_COMMAND="gpg -u $GPG_KEY --no-tty --batch --pinentry-mode loopback"
+ export LOG=$working_dir/$rc_name-$CUR_DATE.log
+ rm -rf "$LOG"
+}
+
+####################################################
+####################################################
+# Following is for packaging and releasing
function reset_release {
info "Reset release folder"
cd ${source_code_folder}
- git reset --hard HEAD~5
+ git reset --hard HEAD~3
git pull -r origin "$GIT_BRANCH"
mvn clean
mvn release:clean
+
+ cd ${packaging_folder}
+ git reset --hard HEAD~3
+ git pull -r origin "$GIT_BRANCH"
+ mvn clean
+
# Update current script
- mv /root/release-publish.sh .release-publish.sh.bak
- cp $source_code_folder/build/release/release-pipeline-docker/release-machine/release-publish.sh /root/
+ mv $rm_dir/release-publish.sh $rm_dir/.release-publish.sh.bak
+ cp $source_code_folder/dev-support/release-manager/release-pipeline-docker/release-machine/bin/release-publish.sh $rm_dir
}
function prepare_release {
+ source /root/.bashrc
+ bash $HOME/.nvm/install.sh
info "Configuration and Clone Code"
git config --global user.name "${GIT_USERNAME}"
- git config --global user.email ${GIT_EMAIL}
- git config --global user.password ${ASF_PASSWORD}
+ git config --global user.email "${GIT_EMAIL}"
mkdir -p $working_dir
cd $working_dir
@@ -215,7 +237,12 @@ function prepare_release {
then
mkdir -p ${source_code_folder}
info "Clone source code to ${source_code_folder} ."
- run_command "Clone Gitbox" git clone "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO" --single-branch --branch "$GIT_BRANCH" ${source_code_folder}
+ run_command "Clone Gitbox" git clone "$FINAL_KYLIN_REPO" --single-branch --branch "$GIT_BRANCH" ${source_code_folder}
+ fi
+
+ if [ ! -d "${packaging_folder}" ]
+ then
+ cp -r ${source_code_folder}/* ${packaging_folder}
fi
if [ ! -d "${release_candidate_folder}" ]
@@ -300,24 +327,24 @@ function publish_release {
function preview_site() {
info "Prepare website"
- if [ ! -d "${document_folder_src}" ]; then
- mkdir -p $document_folder_src
+ if [ ! -d "${document_folder}" ]; then
+ mkdir -p $document_folder
run_command "Install nodejs for docusaurus" switch_node_for_website
fi
- cd $document_folder_src
+ cd $document_folder
if [ ! -d "${document_folder_elder}" ]; then
- run_command "Clone website for kylin4" git clone --single-branch --branch $branch_doc_1 "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO" $branch_doc_1
+ run_command "Clone website for outer" git clone --single-branch --branch $branch_doc_1 $FINAL_KYLIN_REPO outer
else
cd ${document_folder_elder}
- git reset --hard HEAD~4
+ git reset --hard HEAD~2
git pull -r origin $branch_doc_1
fi
if [ ! -d "${document_folder_newer}" ]; then
- run_command "Clone website for kylin5" git clone --single-branch --branch $branch_doc_2 "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO" $branch_doc_2
+ run_command "Clone website for kylin5" git clone --single-branch --branch $branch_doc_2 $FINAL_KYLIN_REPO innner
else
cd ${document_folder_newer}
- git reset --hard HEAD~4
+ git reset --hard HEAD~2
git pull -r origin $branch_doc_2
fi
@@ -330,23 +357,24 @@ function preview_site() {
# Build inner website
cd ${document_folder_newer}/website
- # nvm use 16.14
+ switch_node_for_website
run_command "Install node modules" npm install
run_command "Build inner website" npm run build
document_folder_newer_build=${document_folder_newer}/website/build
# Build outer website
cd ${document_folder_elder}/website
+ switch_node_for_packaging
run_command "Build outer website" jekyll build >>$LOG 2>&1
- document_folder_elder_build=${document_folder_elder}/_site
+ document_folder_elder_build=${document_folder_elder}/website/_site
# Merge two websites
+ run_command "Preview merged website" jekyll s -P 4040 -H 0.0.0.0 -B
+
rm -rf ${document_folder_elder_build}/5.0
mv ${document_folder_newer_build} ${document_folder_elder_build}/5.0
info "Build website should be done, and stored in ${document_folder_elder_build} ."
-
- run_command "Preview merged website" jekyll s -P 7070
- info "Website could be previewed at localhost:7070"
+ info "Website could be previewed at localhost:4040"
}
function publish_site() {
@@ -367,6 +395,56 @@ function publish_site() {
####################################################
# Script running start from here
+if [ $# -eq 0 ]; then
+ exit_with_usage
+else
+ export GITHUB_UID=apache
+ export GIT_BRANCH=kylin5
+ export MODE_RELEASE=0
+ export MODE_OFFICIAL=0
+ export MODE_VERBOSE=0
+ export DRY_RUN='false'
+ while getopts "b:g:ordv" opt; do
+ case $opt in
+ b)
+ export GIT_BRANCH=$OPTARG
+ echo "GIT_BRANCH set to $GIT_BRANCH" ;;
+ d)
+ export DRY_RUN='true'
+ echo "DRY_RUN set to $DRY_RUN" ;;
+ g)
+ export GITHUB_UID=$OPTARG
+ echo "GITHUB_UID set to $GITHUB_UID" ;;
+ o)
+ export MODE_OFFICIAL=1
+ echo "MODE_OFFICIAL set to $MODE_OFFICIAL" ;;
+ r)
+ export MODE_RELEASE=1
+ echo "MODE_RELEASE set to $MODE_RELEASE" ;;
+ v)
+ export MODE_VERBOSE=1
+ echo "MODE_VERBOSE set to $MODE_VERBOSE" ;;
+ \?) error "Invalid option: $OPTARG" ;;
+ esac
+ done
+
+ COMMAND=${@: -1}
+ if [ "$COMMAND" = "reset" ] || \
+ [ "$COMMAND" = "prepare" ] || \
+ [ "$COMMAND" = "publish-snapshot" ] || \
+ [ "$COMMAND" = "publish-release" ] || \
+ [ "$COMMAND" = "preview-site" ] || \
+ [ "$COMMAND" = "publish-site" ];
+ then
+ ask_confirm "You are running step [$COMMAND]"
+ RELEASE_STEP=$COMMAND
+ else
+ echo "Your input $COMMAND is not valid."
+ exit_with_usage
+ fi
+fi
+
+configure_release
prepare_release
if [[ "$RELEASE_STEP" == "reset" ]]; then
diff --git a/dev-support/release-manager/release-pipeline-docker/release-machine/create-release-machine.sh b/dev-support/release-manager/release-pipeline-docker/release-machine/create-release-machine.sh
index ff1da27fc2..71182908d2 100644
--- a/dev-support/release-manager/release-pipeline-docker/release-machine/create-release-machine.sh
+++ b/dev-support/release-manager/release-pipeline-docker/release-machine/create-release-machine.sh
@@ -36,7 +36,7 @@ docker image rm release-machine:latest
docker build -f Dockerfile_2 -t release-machine:latest .
docker image tag release-machine:latest apachekylin/release-machine:latest
-docker push apachekylin/release-machine:latest
+#docker push apachekylin/release-machine:latest
#docker login -u xiaoxiangyu
#docker push apachekylin/release-machine:latest
diff --git a/dev-support/release-manager/release-pipeline-docker/release-machine/run-release-machine.sh b/dev-support/release-manager/release-pipeline-docker/release-machine/run-release-machine.sh
new file mode 100644
index 0000000000..eda9a44244
--- /dev/null
+++ b/dev-support/release-manager/release-pipeline-docker/release-machine/run-release-machine.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+#
+# /*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#
+
+docker stop release-machine
+docker container rm release-machine
+bash create-release-machine.sh
+
+
+RM_WORKDIR=~/release-manager/working-dir
+SOURCE_DIR=~/release-manager/kylin-folder
+source ~/.apache_secret.sh
+mkdir -p $RM_WORKDIR
+
+docker run -d \
+ --name release-machine \
+ --hostname release-machine \
+ --interactive \
+ --volume $RM_WORKDIR:/root/release-manager \
+ --volume $SOURCE_DIR:/root/release-manager/kylin-folder \
+ --env GIT_USERNAME='XiaoxiangYu' \
+ --env GPG_KEY='$GPG_KEY' \
+ --env GPG_PASSPHRASE='$GPG_PASSPHRASE' \
+ --env ASF_USERNAME='$ASF_USERNAME' \
+ --env ASF_PASSWORD='$ASF_PASSWORD' \
+ --env GIT_BRANCH='kylin5' \
+ --env GITHUB_UID='hit-lacus' \
+ --env RELEASE_VERSION='5.0.0' \
+ --env NEXT_RELEASE_VERSION='5.0.1' \
+ --env RC_NUMBER=0 \
+ --env GIT_EMAIL='xxyu@apache.org' \
+ -p 4040:4040 \
+ --tty \
+ release-machine:latest \
+ bash
+docker logs --follow release-machine
+
+docker exec -it release-machine bash
\ No newline at end of file
diff --git a/dev-support/release-manager/release-process-definition.md b/dev-support/release-manager/release-process-definition.md
index bccb2caeb6..92dade52d4 100644
--- a/dev-support/release-manager/release-process-definition.md
+++ b/dev-support/release-manager/release-process-definition.md
@@ -1,13 +1,16 @@
-## Definition of release process
+## Definition of release process for 5.X
### All Steps
-- [ ] Release maven artificial
-- [ ] Release source code
-- [ ] Release binary
-- [ ] Call a vote for RC
+- [ ] Check all issue of this release version whether they have been resolved
+- [ ] Code freeze and running CI to make sure it succeed.
+- [ ] Release maven artifacts using maven-release-plugin
+- [ ] Release source code to svn(candidate path)
+- [ ] Release binary to svn(candidate path)
+- [ ] Call a vote for current RC in dev mailing list
- [ ] Announce RC 's result of a vote
-- [ ] Upload release
-- [ ] Publish website
-- [ ] Announce release
-- [ ] Build and publish standalone docker image for release
\ No newline at end of file
+- [ ] Close all related issues
+- [ ] Upload release to svn(release path)
+- [ ] Update website, including release notes and downloads link(checksum, signature etc)
+- [ ] Announce release in mailing list(user/dev)
+- [ ] (Optional for small release) Build and publish standalone docker image for this release
\ No newline at end of file
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin b/dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin
index 2f047b36fd..e76aea2715 100644
--- a/dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin
+++ b/dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin
@@ -42,7 +42,7 @@ RUN tar -zxf /home/kylin/apache-kylin-$KYLIN_VERSION-*.tar.gz \
COPY conf/kylin/kylin.properties.override $KYLIN_HOME/conf/
-COPY ./entrypoint.sh /home/kylin/entrypoint.sh
+COPY ./scripts/entrypoint.sh /home/kylin/entrypoint.sh
RUN chmod u+x /home/kylin/entrypoint.sh
ENTRYPOINT ["/home/kylin/entrypoint.sh"]
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/README.md b/dev-support/release-manager/standalone-docker/all_in_one/README.md
new file mode 100644
index 0000000000..88dd86b7db
--- /dev/null
+++ b/dev-support/release-manager/standalone-docker/all_in_one/README.md
@@ -0,0 +1,297 @@
+# Preview latest Kylin (5.x)
+
+## [Image Tag Information](https://hub.docker.com/r/apachekylin/apache-kylin-standalone)
+| Tag | Image Contents | Comment & Publish Date |
+|----------------------|----------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 5.0-beta | [**Recommended for users**] The official 5.0.0-beta with Spark bundled. | Uploaded at 2023-09-08, worked fine on Docker Desktop Mac 4.3.0 & 4.22.1(and Windows) , |
+| kylin-4.0.1-mondrian | The official Kylin 4.0.1 with **MDX** function enabled | Uploaded at 2022-05-13 |
+| 5-dev | [**For developer only**] Kylin 5.X package with some sample data/tools etc | Uploaded at 2023-11-21, this image for developer to debug and test Kylin 5.X source code if he/her didn't have a Hadoop env |
+| 5.x-base-dev-only | [**For maintainer only**] Hadoop, Hive, Zookeeper, MySQL, JDK8 | Uploaded at 2023-09-07, this is the base image for all Kylin 5.X image, so it didn't contain Kylin package, see file `Dockerfile_hadoop` for information |
+
+## Why you need Kylin 5
+
+These are the highlight features of Kylin 5, if you are interested, please visit https://kylin.apache.org/5.0/ for detail information.
+
+#### More flexible and enhanced data model
+- Allow adding new dimensions and measures to the existing data model
+- The model adapts to table schema changes while retaining the existing index at the best effort
+- Support last-mile data transformation using Computed Column
+- Support raw query (non-aggregation query) using Table Index
+- Support changing dimension table (SCD2)
+#### Simplified metadata design
+- Merge DataModel and CubeDesc into new DataModel
+- Add DataFlow for more generic data sequence, e.g. streaming alike data flow
+- New metadata AuditLog for better cache synchronization
+#### More flexible index management (was cuboid)
+- Add IndexPlan to support flexible index management
+- Add IndexEntity to support different index type
+- Add LayoutEntity to support different storage layouts of the same Index
+#### Toward a native and vectorized query engine
+- Experiment: Integrate with a native execution engine, leveraging Gluten
+- Support async query
+- Enhance cost-based index optimizer
+#### More
+- Build engine refactoring and performance optimization
+- New WEB UI based on Vue.js, a brand new front-end framework, to replace AngularJS
+- Smooth modeling process in one canvas
+
+### Attention!
+After the time of release of Kylin 5.0.0 , **Kylin 4.X and older version** will be set in **retired** status(NOT in active development).
+
+## How to preview Kylin 5
+
+Deploy a Kylin 5.X instance without any pre-deployed hadoop component by following command:
+
+```shell
+docker run -d \
+ --name Kylin5-Machine \
+ --hostname Kylin5-Machine \
+ -m 8G \
+ -p 7070:7070 \
+ -p 8088:8088 \
+ -p 9870:9870 \
+ -p 8032:8032 \
+ -p 8042:8042 \
+ -p 2181:2181 \
+ apachekylin/apache-kylin-standalone:5.0-beta
+
+docker logs --follow Kylin5-Machine
+```
+
+When you enter these two commands, the logs will scroll
+out in terminal and the process will continue for 3-5 minutes.
+
+```
+
+===============================================================================
+*******************************************************************************
+|
+| Start MySQL at Fri Sep 8 03:35:26 UTC 2023
+| Command: service mysql start
+|
+ * Starting MySQL database server mysqld
+su: warning: cannot change directory to /nonexistent: No such file or directory
+ ...done.
+[Start MySQL] succeed.
+
+===============================================================================
+*******************************************************************************
+|
+| Create Database at Fri Sep 8 03:35:35 UTC 2023
+| Command: mysql -uroot -p123456 -e CREATE DATABASE IF NOT EXISTS kylin default charset utf8mb4 COLLATE utf8mb4_general_ci;
+|
+mysql: [Warning] Using a password on the command line interface can be insecure.
+[Create Database] succeed.
+
+===============================================================================
+*******************************************************************************
+|
+| Init Hive at Fri Sep 8 03:35:35 UTC 2023
+| Command: schematool -initSchema -dbType mysql
+|
+SLF4J: Class path contains multiple SLF4J bindings.
+
+
+...
+...
+
+
+===============================================================================
+*******************************************************************************
+|
+| Start Kylin Instance at Fri Sep 8 03:38:13 UTC 2023
+| Command: /home/kylin/apache-kylin-5.0.0-beta-bin/bin/kylin.sh -v start
+|
+Turn on verbose mode.
+java is /usr/lib/jvm/java-1.8.0-openjdk-amd64/bin/java
+Starting Kylin...
+This user don't have permission to run crontab.
+
+Kylin is checking installation environment, log is at /home/kylin/apache-kylin-5.0.0-beta-bin/logs/check-env.out
+
+Checking Kerberos
+...................................................[SKIP]
+Checking OS Commands
+...................................................[PASS]
+Checking Hadoop Configuration
+...................................................[PASS]
+Checking Permission of HDFS Working Dir
+...................................................[PASS]
+Checking Java Version
+...................................................[PASS]
+Checking Kylin Config
+...................................................[PASS]
+Checking Ports Availability
+...................................................[PASS]
+Checking Spark Driver Host
+...................................................[WARN]
+WARNING:
+ Current kylin_engine_deploymode is 'client'.
+ WARN: 'kylin.storage.columnar.spark-conf.spark.driver.host' is missed, it may cause some problems.
+ WARN: 'kylin.engine.spark-conf.spark.driver.host' is missed, it may cause some problems.
+Checking Spark Dir
+...................................................[PASS]
+Checking Spark Queue
+...................................................[SKIP]
+Checking Spark Availability
+...................................................[PASS]
+Checking Metadata Accessibility
+...................................................[PASS]
+Checking Zookeeper Role
+...................................................[PASS]
+Checking Query History Accessibility
+...................................................[PASS]
+
+> WARN: Command lsb_release is not accessible. Please run on Linux OS.
+> WARN: Command 'lsb_release -a' does not work. Please run on Linux OS.
+> WARN: 'dfs.client.read.shortcircuit' is not enabled which could impact query performance. Check /home/kylin/apache-kylin-5.0.0-beta-bin/hadoop_conf/hdfs-site.xml
+> Available YARN RM cores: 8
+> Available YARN RM memory: 8192M
+> The max executor instances can be 8
+> The current executor instances is 1
+Checking environment finished successfully. To check again, run 'bin/check-env.sh' manually.
+
+KYLIN_HOME is:/home/kylin/apache-kylin-5.0.0-beta-bin
+KYLIN_CONFIG_FILE is:/home/kylin/apache-kylin-5.0.0-beta-bin/conf/kylin.properties
+SPARK_HOME is:/home/kylin/apache-kylin-5.0.0-beta-bin/spark
+Retrieving hadoop config dir...
+KYLIN_JVM_SETTINGS is -server -Xms1g -Xmx8g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:G1HeapRegionSize=16m -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -Xloggc:/home/kylin/apache-kylin-5.0.0-beta-bin/logs/kylin.gc.%p -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M -XX:-OmitStackTraceInFastThrow -Dlog4j [...]
+KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging
+KYLIN_LD_LIBRARY_SETTINGS is not set, it is okay unless you want to specify your own native path
+SPARK_HDP_VERSION is set to 'hadoop'
+Export SPARK_HOME to /home/kylin/apache-kylin-5.0.0-beta-bin/spark
+Checking Zookeeper role...
+Checking Spark directory...
+KYLIN_HOME is:/home/kylin/apache-kylin-5.0.0-beta-bin
+KYLIN_CONFIG_FILE is:/home/kylin/apache-kylin-5.0.0-beta-bin/conf/kylin.properties
+SPARK_HOME is:/home/kylin/apache-kylin-5.0.0-beta-bin/spark
+KYLIN_JVM_SETTINGS is -server -Xms1g -Xmx8g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:G1HeapRegionSize=16m -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -Xloggc:/home/kylin/apache-kylin-5.0.0-beta-bin/logs/kylin.gc.%p -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M -XX:-OmitStackTraceInFastThrow -Dlog4j [...]
+KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging
+KYLIN_LD_LIBRARY_SETTINGS is not set, it is okay unless you want to specify your own native path
+SPARK_HDP_VERSION is set to 'hadoop'
+Export SPARK_HOME to /home/kylin/apache-kylin-5.0.0-beta-bin/spark
+KYLIN_HOME is:/home/kylin/apache-kylin-5.0.0-beta-bin
+KYLIN_CONFIG_FILE is:/home/kylin/apache-kylin-5.0.0-beta-bin/conf/kylin.properties
+SPARK_HOME is:/home/kylin/apache-kylin-5.0.0-beta-bin/spark
+KYLIN_JVM_SETTINGS is -server -Xms1g -Xmx8g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:G1HeapRegionSize=16m -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintAdaptiveSizePolicy -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -Xloggc:/home/kylin/apache-kylin-5.0.0-beta-bin/logs/kylin.gc.%p -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M -XX:-OmitStackTraceInFastThrow -Dlog4j [...]
+KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging
+KYLIN_LD_LIBRARY_SETTINGS is not set, it is okay unless you want to specify your own native path
+SPARK_HDP_VERSION is set to 'hadoop'
+Export SPARK_HOME to /home/kylin/apache-kylin-5.0.0-beta-bin/spark
+Kylin is starting. It may take a while. For status, please visit http://Kylin5-Machine:7070/kylin/index.html.
+You may also check status via: PID:9781, or Log: /home/kylin/apache-kylin-5.0.0-beta-bin/logs/kylin.log.
+[Start Kylin Instance] succeed.
+Checking Check Env Script's status...
+/home/kylin/apache-kylin-5.0.0-beta-bin/bin/check-env-bypass
++
+Check Check Env Script succeed.
+0
+Kylin service is already available for you to preview.
+```
+
+Finally, the following message indicates that the Kylin is ready :
+
+```
+Kylin service is already available for you to preview.
+```
+
+After that, please press `Ctrl + C` to exit `docker logs`, and visit Kylin web UI.
+
+
+| Service Name | URL |
+|--------------|-----------------------------|
+| Kylin | http://localhost:7070/kylin |
+| Yarn | http://localhost:8088 |
+| HDFS | http://localhost:9870 |
+
+When you log in Kylin web UI, please remember your username is **ADMIN** , and password is **KYLIN** .
+
+
+To stop and remove the container, use these commands.
+```
+docker stop Kylin5-Machine
+docker rm Kylin5-Machine
+```
+
+
+### Notes
+
+If you are using mac docker desktop, please ensure that you have set Resources: Memory=8GB and Cores=6 cores at least,
+so that can run kylin standalone on docker well.
+
+If you are interested in `Dockerfile`, please visit https://github.com/apache/kylin/blob/kylin5/dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin .
+
+If you want to configure and restart Kylin instance,
+you can use `docker exec -it Kylin5-Machine bash` to login the container.
+Kylin is deployed at `/home/kylin/apache-kylin-{VERSION}-bin`.
+
+If you find some issues, please send email to Kylin's user mailing list.
+
+For how to connect to PowerBI, here is [a discussion in mailing list](https://lists.apache.org/thread/74pxjcx58t3m83r6o9b1hrzjjd40lhy4) .
+
+---------
+# Preview Inactive version (4.x & 3.x)
+
+After the time of release of Kylin 5.0.0 , **Kylin 4.X and older version** will be set in **retired** status(NOT in active development),
+following is the command for Kylin 4.X:
+
+```
+docker run -d \
+-m 8G \
+-p 7070:7070 \
+-p 8088:8088 \
+-p 50070:50070 \
+-p 8032:8032 \
+-p 8042:8042 \
+-p 2181:2181 \
+apachekylin/apache-kylin-standalone:4.0.0
+```
+
+and the command for Kylin 4.X with mondrian:
+
+```
+docker run -d \
+-m 8G \
+-p 7070:7070 \
+-p 7080:7080 \
+-p 8088:8088 \
+-p 50070:50070 \
+-p 8032:8032 \
+-p 8042:8042 \
+-p 2181:2181 \
+apachekylin/apache-kylin-standalone:kylin-4.0.1-mondrian
+```
+
+and the command for Kylin 3.X:
+
+```sh
+docker run -d \
+-m 8G \
+-p 7070:7070 \
+-p 8088:8088 \
+-p 50070:50070 \
+-p 8032:8032 \
+-p 8042:8042 \
+-p 16010:16010 \
+apachekylin/apache-kylin-standalone:3.1.0
+```
+
+-----
+
+## Note
+
+If you are using mac docker desktop, please ensure that you have set Resources: Memory=8GB and Cores=6 cores at least so that can run kylin standalone on docker well.
+
+如果是使用 mac docker desktop 的用户，请将 docker desktop 中 Resource 的内存至少设置为 8gb 以及 6 core，以保证能流畅运行 kylin standalone on docker.
+
+-----
+
+For user, please visit http://kylin.apache.org/docs/install/kylin_docker.html for detail.
+
+对于中国用户，请参阅 http://kylin.apache.org/cn/docs/install/kylin_docker.html
+
+------
+
+JIRA : https://issues.apache.org/jira/browse/KYLIN-4114.
+
+For any suggestion, please contact us via Kylin's mailing list: user@kylin.apache.org.
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/build_image.sh b/dev-support/release-manager/standalone-docker/all_in_one/build_and_run.sh
similarity index 80%
copy from dev-support/release-manager/standalone-docker/all_in_one/build_image.sh
copy to dev-support/release-manager/standalone-docker/all_in_one/build_and_run.sh
index a3ffa76fca..fa32c8d22b 100755
--- a/dev-support/release-manager/standalone-docker/all_in_one/build_image.sh
+++ b/dev-support/release-manager/standalone-docker/all_in_one/build_and_run.sh
@@ -16,7 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-TAG=5.0-beta
+TAG=5-dev
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd ${DIR} || exit
@@ -37,23 +37,45 @@ fi
# tar -czf apache-kylin-5.0.0-beta-bin.tar.gz apache-kylin-5.0.0-beta-bin
echo "start to build Hadoop docker image"
+cp ../../tpch-benchmark/scripts/*.sql scripts/
# docker build -f Dockerfile_hadoop -t hadoop3.2.1-all-in-one-for-kylin5 .
docker build -f Dockerfile_kylin -t apachekylin/apache-kylin-standalone:${TAG} .
+BUILD_RESULT=$?
+
+if [ "$BUILD_RESULT" != "0" ]; then
+ echo "Image build failed, please check"
+ exit 1
+fi
+echo "Image build succeed"
docker image tag docker.io/apachekylin/apache-kylin-standalone:${TAG} apachekylin/apache-kylin-standalone:${TAG}
+
+echo "Start this image locally, and push it to dockerhub later."
docker stop Kylin5-Machine
docker rm Kylin5-Machine
docker run -d \
--name Kylin5-Machine \
--hostname Kylin5-Machine \
- -m 8G \
+ -m 15G \
-p 7070:7070 \
-p 8088:8088 \
- -p 50070:50070 \
+ -p 9870:9870 \
-p 8032:8032 \
-p 8042:8042 \
-p 2181:2181 \
+ -p 3306:3306 \
+ -p 9000:9000 \
+ -p 9864:9864 \
+ -p 9866:9866 \
+ -p 9867:9867 \
+ -p 8030:8030 \
+ -p 8031:9867 \
+ -p 8033:9867 \
+ -p 8040:9867 \
+ -p 8040:9867 \
+
+
apachekylin/apache-kylin-standalone:${TAG}
-docker logs --follow Kylin5-Machine
+docker logs --follow Kylin5-Machine
\ No newline at end of file
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/capacity-scheduler.xml b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/capacity-scheduler.xml
new file mode 100644
index 0000000000..8f016e2c3b
--- /dev/null
+++ b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/capacity-scheduler.xml
@@ -0,0 +1,134 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-applications</name>
+ <value>4</value>
+ <description>
+ Maximum number of applications that can be pending and running.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.5</value>
+ <description>
+ Maximum percent of resources in the cluster which can be used to run
+ application masters i.e. controls number of concurrent running
+ applications.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.resource-calculator</name>
+ <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+ <description>
+ The ResourceCalculator implementation to be used to compare
+ Resources in the scheduler.
+ The default i.e. DefaultResourceCalculator only uses Memory while
+ DominantResourceCalculator uses dominant-resource to compare
+ multi-dimensional resources such as Memory, CPU etc.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ <value>default</value>
+ <description>
+ The queues at the this level (root is the root queue).
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.capacity</name>
+ <value>100</value>
+ <description>Default queue target capacity.</description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
+ <value>1</value>
+ <description>
+ Default queue user limit a percentage from 0.0 to 1.0.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
+ <value>100</value>
+ <description>
+ The maximum capacity of the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.state</name>
+ <value>RUNNING</value>
+ <description>
+ The state of the default queue. State can be one of RUNNING or STOPPED.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
+ <value>*</value>
+ <description>
+ The ACL of who can submit jobs to the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
+ <value>*</value>
+ <description>
+ The ACL of who can administer jobs on the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.node-locality-delay</name>
+ <value>40</value>
+ <description>
+ Number of missed scheduling opportunities after which the CapacityScheduler
+ attempts to schedule rack-local containers.
+ Typically this should be set to number of nodes in the cluster, By default is setting
+ approximately number of nodes in one rack which is 40.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.queue-mappings</name>
+ <value></value>
+ <description>
+ A list of mappings that will be used to assign jobs to queues
+ The syntax for this list is [u|g]:[name]:[queue_name][,next mapping]*
+ Typically this list will be used to map users to queues,
+ for example, u:%user:%user maps all users to queues with the same name
+ as the user.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.queue-mappings-override.enable</name>
+ <value>false</value>
+ <description>
+ If a queue mapping is present, will it override the value specified
+ by the user? This can be used by administrators to place jobs in queues
+ that are different than the one specified by the user.
+ The default is false.
+ </description>
+ </property>
+
+</configuration>
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/core-site.xml b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/core-site.xml
new file mode 100644
index 0000000000..2e7e4a5671
--- /dev/null
+++ b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/core-site.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>hadoop.tmp.dir</name>
+ <value>/data/hadoop</value>
+ <description>Abase for other temporary directories.</description>
+ </property>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://kylin5-machine:9000</value>
+ </property>
+</configuration>
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/hdfs-site.xml b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/hdfs-site.xml
new file mode 100644
index 0000000000..cd4edeedd3
--- /dev/null
+++ b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/hdfs-site.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>dfs.datanode.address</name>
+ <value>kylin5-machine:9866</value>
+ </property>
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>/data/hadoop/dfs/name</value>
+ </property>
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/data/hadoop/dfs/data</value>
+ </property>
+</configuration>
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/mapred-site.xml b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/mapred-site.xml
new file mode 100644
index 0000000000..ac8ef33969
--- /dev/null
+++ b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/mapred-site.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/yarn-site.xml b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/yarn-site.xml
new file mode 100644
index 0000000000..809688d8fa
--- /dev/null
+++ b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop-dev/yarn-site.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+
+ <property>
+ <name>yarn.nodemanager.hostname</name>
+ <value>kylin5-machine</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.hostname</name>
+ <value>kylin5-machine</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>6144</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>14336</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>7</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.6</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+ <value>98.5</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-address</name>
+ <value>localhost:2181</value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/conf/kylin/kylin.properties.override b/dev-support/release-manager/standalone-docker/all_in_one/conf/kylin-dev/kylin.properties.override
similarity index 73%
copy from dev-support/release-manager/standalone-docker/all_in_one/conf/kylin/kylin.properties.override
copy to dev-support/release-manager/standalone-docker/all_in_one/conf/kylin-dev/kylin.properties.override
index b1757f9810..b5acf325d6 100644
--- a/dev-support/release-manager/standalone-docker/all_in_one/conf/kylin/kylin.properties.override
+++ b/dev-support/release-manager/standalone-docker/all_in_one/conf/kylin-dev/kylin.properties.override
@@ -15,45 +15,61 @@
# limitations under the License.
#
+# The http session will timeout in 3 hours
+spring.session.store-type=JDBC
+spring.session.timeout=10800
+
+# Turn off query cache
+kylin.query.cache-enabled=false
+
server.port=7070
kylin.query.init-sparder-async=false
kylin.env.apache-hadoop-conf-dir=/opt/hadoop-3.2.1/etc/hadoop
kylin.env.apache-hive-conf-dir=/opt/apache-hive-3.1.2-bin/conf
+
+# The metastore connection information
kylin.metadata.url=kylin@jdbc,driverClassName=com.mysql.jdbc.Driver,url=jdbc:mysql://localhost:3306/kylin?useUnicode=true&characterEncoding=utf8,username=root,password=123456,maxTotal=50,maxIdle=8
+
kylin.env.zookeeper-connect-string=localhost:2181
kylin.env.hdfs-working-dir=/kylin
-# Query
+# The yarn resource used by query engine(spark session)
kylin.storage.columnar.spark-conf.spark.driver.memory=512M
kylin.storage.columnar.spark-conf.spark.driver.memoryOverhead=256M
kylin.storage.columnar.spark-conf.spark.executor.cores=1
kylin.storage.columnar.spark-conf.spark.executor.instances=1
-kylin.storage.columnar.spark-conf.spark.executor.memory=1024M
+kylin.storage.columnar.spark-conf.spark.executor.memory=2048M
kylin.storage.columnar.spark-conf.spark.executor.memoryOverhead=256M
-# Build
+# The yarn resource used by build engine(spark session)
kylin.engine.driver-memory-strategy=2,500
kylin.engine.spark-conf.spark.driver.memory=512M
kylin.engine.spark-conf.spark.driver.memoryOverhead=256M
kylin.engine.spark-conf.spark.executor.cores=1
-kylin.engine.spark-conf.spark.executor.instances=1
-kylin.engine.spark-conf.spark.executor.memory=1024M
+kylin.engine.spark-conf.spark.executor.instances=2
+kylin.engine.spark-conf.spark.executor.memory=2048M
kylin.engine.spark-conf.spark.executor.memoryOverhead=512M
-# Async Query
+# The yarn resource used by query engine(spark session) Async Query
kylin.query.async-query.spark-conf.spark.executor.cores=1
kylin.query.async-query.spark-conf.spark.driver.memory=1024M
kylin.query.async-query.spark-conf.spark.executor.memory=1024M
kylin.query.async-query.spark-conf.spark.executor.instances=1
kylin.query.async-query.spark-conf.spark.executor.memoryOverhead=512M
-# Other
+# Turn off the password force reset function
kylin.metadata.random-admin-password.enabled=false
+
kylin.query.engine.push-down.enable-prepare-statement-with-params=true
kylin.query.calcite.extras-props.FUN=standard,oracle
kylin.circuit-breaker.threshold.project=500
kylin.engine.resource-request-over-limit-proportion=3.0
-# Shutdown the lock
+# Kylin can only run two jobs at the same time
+kylin.job.max-concurrent-jobs=2
+
+# If this switch is turned on, Kylin will check node's available memory, if remained
+# memory is not fulfilled, job will be hanged.
+# In some cases, the Kylin will refuse to start any build job by this check logic.
kylin.job.auto-set-concurrent-jobs=false
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/conf/kylin/kylin.properties.override b/dev-support/release-manager/standalone-docker/all_in_one/conf/kylin/kylin.properties.override
index b1757f9810..5c6cfc0a75 100644
--- a/dev-support/release-manager/standalone-docker/all_in_one/conf/kylin/kylin.properties.override
+++ b/dev-support/release-manager/standalone-docker/all_in_one/conf/kylin/kylin.properties.override
@@ -15,45 +15,61 @@
# limitations under the License.
#
+# The http session will timeout in 3 hours
+spring.session.store-type=JDBC
+spring.session.timeout=10800
+
+# Turn off query cache
+kylin.query.cache-enabled=false
+
server.port=7070
kylin.query.init-sparder-async=false
kylin.env.apache-hadoop-conf-dir=/opt/hadoop-3.2.1/etc/hadoop
kylin.env.apache-hive-conf-dir=/opt/apache-hive-3.1.2-bin/conf
+
+# The metastore connection information
kylin.metadata.url=kylin@jdbc,driverClassName=com.mysql.jdbc.Driver,url=jdbc:mysql://localhost:3306/kylin?useUnicode=true&characterEncoding=utf8,username=root,password=123456,maxTotal=50,maxIdle=8
+
kylin.env.zookeeper-connect-string=localhost:2181
kylin.env.hdfs-working-dir=/kylin
-# Query
+# The yarn resource used by query engine(spark session)
kylin.storage.columnar.spark-conf.spark.driver.memory=512M
kylin.storage.columnar.spark-conf.spark.driver.memoryOverhead=256M
kylin.storage.columnar.spark-conf.spark.executor.cores=1
kylin.storage.columnar.spark-conf.spark.executor.instances=1
-kylin.storage.columnar.spark-conf.spark.executor.memory=1024M
+kylin.storage.columnar.spark-conf.spark.executor.memory=2048M
kylin.storage.columnar.spark-conf.spark.executor.memoryOverhead=256M
-# Build
+# The yarn resource used by build engine(spark session)
kylin.engine.driver-memory-strategy=2,500
kylin.engine.spark-conf.spark.driver.memory=512M
kylin.engine.spark-conf.spark.driver.memoryOverhead=256M
kylin.engine.spark-conf.spark.executor.cores=1
kylin.engine.spark-conf.spark.executor.instances=1
-kylin.engine.spark-conf.spark.executor.memory=1024M
+kylin.engine.spark-conf.spark.executor.memory=2048M
kylin.engine.spark-conf.spark.executor.memoryOverhead=512M
-# Async Query
+# The yarn resource used by query engine(spark session) Async Query
kylin.query.async-query.spark-conf.spark.executor.cores=1
kylin.query.async-query.spark-conf.spark.driver.memory=1024M
kylin.query.async-query.spark-conf.spark.executor.memory=1024M
kylin.query.async-query.spark-conf.spark.executor.instances=1
kylin.query.async-query.spark-conf.spark.executor.memoryOverhead=512M
-# Other
+# Turn off the password force reset function
kylin.metadata.random-admin-password.enabled=false
+
kylin.query.engine.push-down.enable-prepare-statement-with-params=true
kylin.query.calcite.extras-props.FUN=standard,oracle
kylin.circuit-breaker.threshold.project=500
kylin.engine.resource-request-over-limit-proportion=3.0
-# Shutdown the lock
+# Kylin can only run two jobs at the same time
+kylin.job.max-concurrent-jobs=2
+
+# If this switch is turned on, Kylin will check node's available memory, if remained
+# memory is not fulfilled, job will be hanged.
+# In some cases, the Kylin will refuse to start any build job by this check logic.
kylin.job.auto-set-concurrent-jobs=false
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/conf/mysql/my.cnf b/dev-support/release-manager/standalone-docker/all_in_one/conf/mysql/my.cnf
new file mode 100644
index 0000000000..a4e30476d6
--- /dev/null
+++ b/dev-support/release-manager/standalone-docker/all_in_one/conf/mysql/my.cnf
@@ -0,0 +1,4 @@
+[mysql]
+default-character-set=utf8mb4
+[client]
+default-character-set=utf8mb4
\ No newline at end of file
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin b/dev-support/release-manager/standalone-docker/all_in_one/dev-docker/Dockerfile_kylin_dev
similarity index 59%
copy from dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin
copy to dev-support/release-manager/standalone-docker/all_in_one/dev-docker/Dockerfile_kylin_dev
index 2f047b36fd..43b0dcdfeb 100644
--- a/dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin
+++ b/dev-support/release-manager/standalone-docker/all_in_one/dev-docker/Dockerfile_kylin_dev
@@ -22,27 +22,23 @@ FROM apachekylin/apache-kylin-standalone:5.x-base-dev-only
USER root
RUN apt-get -y install curl
-RUN apt-get update && apt-get -y install openjdk-8-jdk
+RUN apt-get update && apt-get -y install openjdk-8-jdk unzip iputils-ping net-tools
# make a new workdir
RUN mkdir -p /home/kylin
+RUN mkdir -p /home/kylin/data
+RUN mkdir -p /home/kylin/scripts
-# change workdir to install Kylin
WORKDIR /home/kylin
-ENV KYLIN_VERSION 5.0.0-beta
-ENV KYLIN_HOME /home/kylin/apache-kylin-$KYLIN_VERSION-bin
+ENV HADOOP_CONF_DIR /opt/hadoop-3.2.1/etc/hadoop
-COPY package/apache-kylin-$KYLIN_VERSION-*.tar.gz /home/kylin/
+COPY conf/hadoop-dev/* $HADOOP_CONF/
+COPY dev-docker/*.sql /home/kylin/scripts/
+COPY conf/mysql/my.cnf /etc/mysql/conf.d/
+COPY package/tpch-sf2.zip /home/kylin/scripts/
-RUN tar -zxf /home/kylin/apache-kylin-$KYLIN_VERSION-*.tar.gz \
- && rm -f /home/kylin/apache-kylin-$KYLIN_VERSION-*.tar.gz \
- && cp $HIVE_HOME/lib/mysql-connector-java.jar $KYLIN_HOME/lib/ext/ \
- && cp $HIVE_HOME/lib/mysql-connector-java.jar $KYLIN_HOME/spark/hive_1_2_2/
+COPY dev-docker/entrypoint-dev.sh /home/kylin/entrypoint-dev.sh
+RUN chmod u+x /home/kylin/entrypoint-dev.sh
-COPY conf/kylin/kylin.properties.override $KYLIN_HOME/conf/
-
-COPY ./entrypoint.sh /home/kylin/entrypoint.sh
-RUN chmod u+x /home/kylin/entrypoint.sh
-
-ENTRYPOINT ["/home/kylin/entrypoint.sh"]
+ENTRYPOINT ["/home/kylin/entrypoint-dev.sh"]
\ No newline at end of file
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/build_image.sh b/dev-support/release-manager/standalone-docker/all_in_one/dev-docker/build_and_run.sh
similarity index 60%
rename from dev-support/release-manager/standalone-docker/all_in_one/build_image.sh
rename to dev-support/release-manager/standalone-docker/all_in_one/dev-docker/build_and_run.sh
index a3ffa76fca..ea4b411440 100755
--- a/dev-support/release-manager/standalone-docker/all_in_one/build_image.sh
+++ b/dev-support/release-manager/standalone-docker/all_in_one/dev-docker/build_and_run.sh
@@ -16,15 +16,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-TAG=5.0-beta
-
+TAG=kylin5-dev
+cd ..
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd ${DIR} || exit
echo "build image in dir "${DIR}
echo "package kylin in local for building image"
-if [[ ! -d ${DIR}/package/ ]]; then
- mkdir -p ${DIR}/package/
+if [[ ! -d ${DIR}/../package/ ]]; then
+ mkdir -p ${DIR}/../package/
fi
# The official package didn't carry with Spark binary,
@@ -37,23 +37,49 @@ fi
# tar -czf apache-kylin-5.0.0-beta-bin.tar.gz apache-kylin-5.0.0-beta-bin
echo "start to build Hadoop docker image"
-# docker build -f Dockerfile_hadoop -t hadoop3.2.1-all-in-one-for-kylin5 .
-docker build -f Dockerfile_kylin -t apachekylin/apache-kylin-standalone:${TAG} .
+cp ../../tpch-benchmark/scripts/*.sql dev-docker
+docker build -f dev-docker/Dockerfile_kylin_dev -t apachekylin/apache-kylin-standalone:${TAG} .
+BUILD_RESULT=$?
+
+if [ "$BUILD_RESULT" != "0" ]; then
+ echo "Image build failed, please check"
+ exit 1
+fi
+echo "Image build succeed"
docker image tag docker.io/apachekylin/apache-kylin-standalone:${TAG} apachekylin/apache-kylin-standalone:${TAG}
-docker stop Kylin5-Machine
-docker rm Kylin5-Machine
+echo "Start this image locally, and push it to dockerhub later."
+
+rm dev-docker/*.sql
+
+docker stop kylin5-dev
+docker rm kylin5-dev
docker run -d \
- --name Kylin5-Machine \
+ --name kylin5-dev \
--hostname Kylin5-Machine \
+ --network host \
-m 8G \
- -p 7070:7070 \
- -p 8088:8088 \
- -p 50070:50070 \
- -p 8032:8032 \
- -p 8042:8042 \
- -p 2181:2181 \
- apachekylin/apache-kylin-standalone:${TAG}
-
-docker logs --follow Kylin5-Machine
+ --env OUTER_HOST="172.16.4.56" \
+ apachekylin/apache-kylin-standalone:kylin5-dev
+
+docker logs --follow kylin5-dev
+
+
+#docker run -d \
+# --name kylin5-dev \
+# --hostname Kylin5-Machine \
+# -p 2181:2181 \
+# -p 3306:3306 \
+# -p 9000:9000 \
+# -p 9866:9866 \
+# -p 8032:8032 \
+# -p 9867:9867 \
+# -m 8G \
+# --env OUTER_HOST="127.0.0.1" \
+# apachekylin/apache-kylin-standalone:kylin5-dev
+#
+#docker logs --follow kylin5-dev
+#
+#docker save apachekylin/apache-kylin-standalone:kylin5-dev | bzip2 | \
+# ssh kylin@worker-03 'bunzip2 | docker load'
\ No newline at end of file
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/entrypoint.sh b/dev-support/release-manager/standalone-docker/all_in_one/dev-docker/entrypoint-dev.sh
similarity index 60%
copy from dev-support/release-manager/standalone-docker/all_in_one/entrypoint.sh
copy to dev-support/release-manager/standalone-docker/all_in_one/dev-docker/entrypoint-dev.sh
index e503e16cf6..14924a5725 100644
--- a/dev-support/release-manager/standalone-docker/all_in_one/entrypoint.sh
+++ b/dev-support/release-manager/standalone-docker/all_in_one/dev-docker/entrypoint-dev.sh
@@ -16,7 +16,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-echo "127.0.0.1 sandbox" >> /etc/hosts
+sed '/127.0.0.1/d' /etc/hosts
+echo "${OUTER_HOST:-127.0.0.1} Kylin5-Machine" >> /etc/hosts
+cat /etc/hosts
START_FLAG="/home/kylin/first_run"
TIMEOUT=600
@@ -40,7 +42,7 @@ function run_command {
sleep 7200
exit $EC
else
- echo "[$STEP] succeed."
+ echo "| $STEP succeed."
fi
}
@@ -75,25 +77,26 @@ rm -f /tmp/*.pid
rm -rf /data/zookeeper/*
rm -f /data/zookeeper/zookeeper_server.pid
+##############################################
+##############################################
##############################################
-run_command "Start MySQL" service mysql start
-if [ ! -f "/home/kylin/first_run" ]
+run_command "Start MySQL server" service mysql start
+if [ ! -f $START_FLAG ]
then
+ sed -i 's/127.0.0.1/0.0.0.0/g' /etc/mysql/mysql.conf.d/mysqld.cnf
+ sed -i 's/localhost/kylin5-machine/g' /opt/hadoop-3.2.1/etc/hadoop/workers
+ mysql -uroot -p123456 mysql -e "update user set host='%' where user='root'; grant all privileges on *.* to root@'%';FLUSH PRIVILEGES;"
run_command "Create Database" mysql -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS kylin default charset utf8mb4 COLLATE utf8mb4_general_ci;"
- run_command "Init Hive" schematool -initSchema -dbType mysql
+ run_command "Init Hive Metastore" schematool -initSchema -dbType mysql
run_command "Format HDFS" hdfs namenode -format
fi
-run_command "Start HDFS [NameNode]" hdfs --daemon start namenode
-run_command "Start HDFS [DataNode]" hdfs --daemon start datanode
-
-# start yarn
-run_command "Start Yarn [ResourceManager]" yarn --daemon start resourcemanager
-run_command "Start Yarn [NodeManager]" yarn --daemon start nodemanager
+run_command "Start HDFS [NameNode]" hdfs --daemon stop namenode; hdfs --daemon start namenode
+run_command "Start HDFS [DataNode]" hdfs --daemon stop datanode; hdfs --daemon start datanode
-# start job history server
-# run_command "" mapred --daemon start historyserver
+run_command "Start Yarn [ResourceManager]" yarn --daemon stop resourcemanager; yarn --daemon start resourcemanager
+run_command "Start Yarn [NodeManager]" yarn --daemon stop nodemanager; yarn --daemon start nodemanager
run_command "Start Zookeeper" "$ZK_HOME"/bin/zkServer.sh start
@@ -104,17 +107,20 @@ then
check_and_monitor_status "HDFS Usability" hadoop fs -mkdir /tmp
mkdir -p "$KYLIN_HOME"/logs
hdfs dfs -mkdir -p /kylin
- run_command "Prepare sample data" "$KYLIN_HOME"/bin/sample.sh
+ if [ -f "/home/kylin/tpch-sf2.zip" ]
+ then
+ cd /home/kylin/scripts || exit
+ unzip tpch-sf2.zip
+ run_command "Update tpch tables comments" service mysql restart; mysql -u root -p123456 hive < /home/kylin/update-hive-column-comment.sql
+ run_command "Create tpch tables" hive -f /home/kylin/create-all-tables.sql
+ hadoop fs -rm -r /user/data/tpch
+ hadoop fs -mkdir /user/data/tpch
+ hadoop fs -copyFromLocal data/* /user/data/tpch
+ fi
fi
-
-run_command "Start Kylin Instance" "$KYLIN_HOME"/bin/kylin.sh -v start
-
-check_and_monitor_status "Check Env Script" ls $KYLIN_HOME/bin/check-env-bypass
-check_and_monitor_status "Kylin Instance" cat "$KYLIN_HOME"/logs/kylin.log | grep -c "Initialized Spark"
-
touch $START_FLAG
-echo "Kylin service is already available for you to preview."
+echo "Kylin-Machine is ready for you to remote debug with your local kylin process."
while :
do
sleep 10
diff --git a/dev-support/release-manager/standalone-docker/all_in_one/entrypoint.sh b/dev-support/release-manager/standalone-docker/all_in_one/scripts/entrypoint.sh
similarity index 100%
rename from dev-support/release-manager/standalone-docker/all_in_one/entrypoint.sh
rename to dev-support/release-manager/standalone-docker/all_in_one/scripts/entrypoint.sh
diff --git a/dev-support/release-manager/tpch-benchmark/README.md b/dev-support/release-manager/tpch-benchmark/README.md
new file mode 100644
index 0000000000..b02372dd20
--- /dev/null
+++ b/dev-support/release-manager/tpch-benchmark/README.md
@@ -0,0 +1,990 @@
+# Modeling of Kylin 5.0.0 for TPC-H
+
+---
+
+# Background
+
+### What is TPC-H ?
+
+
+
+### Query 1 : Pricing Summary Report Query
+
+#### Original Version
+
+
+```sql
+/**
+ * TPC-H Query 1 : Pricing Summary Report Query / 价格统计报告查询
+ * 在单个表lineitem上查询某个时间段内，对已经付款的、已经运送的等各类商品进行统计，包括业务量的计费、发货、折扣、税、平均价格等信息。
+ * 带有分组、排序、聚集操作并存的单表查询操作。这个查询会导致表上的数据有95%到97%行被读取到。
+ */
+select /*+ MODEL_PRIORITY(Lineitem) */
+ l_returnflag,
+ l_linestatus,
+ sum(l_quantity) as sum_qty,
+ sum(l_extendedprice) as sum_base_price,
+ sum(l_extendedprice * (1 - l_discount)) as sum_disc_price,
+ sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge,
+ avg(l_quantity) as avg_qty,
+ avg(l_extendedprice) as avg_price,
+ avg(l_discount) as avg_disc,
+ count(*) as count_order
+from tpch_cn.lineitem
+where l_shipdate <= '1998-12-01' - interval '90' day -- DELTA is randomly selected within [60. 120].
+group by l_returnflag,
+ l_linestatus
+order by l_returnflag,
+ l_linestatus;
+```
+
+#### Modified Version
+
+```sql
+select l_returnflag,
+ l_linestatus,
+ sum(l_quantity) as sum_qty,
+ sum(l_extendedprice) as sum_base_price,
+ sum(l_extendedprice * (1 - l_discount)) as sum_disc_price,
+ sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge,
+ avg(l_quantity) as avg_qty,
+ avg(l_extendedprice) as avg_price,
+ avg(l_discount) as avg_disc,
+ count(*) as count_order
+from tpch_cn.lineitem
+where l_shipdate <= '1998-09-01'
+group by l_returnflag,
+ l_linestatus
+order by l_returnflag,
+ l_linestatus;
+```
+
+
+### Query 2 : Minimum Cost Supplier Query
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 2 : Minimum Cost Supplier Query / 最小代价供货商查询
+ * Return the first 100 selected rows
+ * 得到给定的区域内，对于指定的零件（某一类型和大小的零件），哪个供应者能以最低的价格供应它，就可以选择哪个供应者来订货。
+ * 带有排序、聚集操作、子查询并存的多表查询操作。
+ */
+select s_acctbal,
+ s_name,
+ n_name,
+ p_partkey,
+ p_mfgr,
+ s_address,
+ s_phone,
+ s_comment
+from tpch_cn.part,
+ tpch_cn.supplier,
+ tpch_cn.partsupp,
+ tpch_cn.nation,
+ tpch_cn.region
+where p_partkey = ps_partkey
+ and s_suppkey = ps_suppkey
+ and p_size = 15 -- SIZE is randomly selected within [1. 50];
+ and p_type like '%BRASS' -- TYPE is randomly selected within the list Syllable 3 defined for Types in Clause 4.2.2.13;
+ and s_nationkey = n_nationkey
+ and n_regionkey = r_regionkey
+ and r_name = 'EUROPE' -- REGION is randomly selected within the list of values defined for R_NAME in 4.2.3.
+ and ps_supplycost
+ = ( -- 关连子查询
+ select min(ps_supplycost)
+ from tpch_cn.partsupp,
+ tpch_cn.supplier,
+ tpch_cn.nation,
+ tpch_cn.region
+ where p_partkey = ps_partkey -- p_partkey 是外部的 Join Key
+ and s_suppkey = ps_suppkey
+ and s_nationkey = n_nationkey
+ and n_regionkey = r_regionkey
+ and r_name = 'EUROPE')
+order by s_acctbal desc,
+ n_name,
+ s_name,
+ p_partkey
+limit 100;
+```
+
+#### Modified Version
+```sql
+select s_acctbal,
+ s_name,
+ n_name,
+ p_partkey,
+ p_mfgr,
+ s_address,
+ s_phone,
+ s_comment
+from
+ tpch_cn.partsupp,
+ tpch_cn.part,
+ tpch_cn.supplier,
+ tpch_cn.nation,
+ tpch_cn.region
+where p_partkey = ps_partkey
+ and s_suppkey = ps_suppkey
+ and p_size = 15
+ and p_type like '%BRASS'
+ and s_nationkey = n_nationkey
+ and n_regionkey = r_regionkey
+ and r_name = 'EUROPE'
+ and ps_supplycost
+ = (
+ select min(ps_supplycost)
+ from tpch_cn.partsupp,
+ tpch_cn.supplier,
+ tpch_cn.nation,
+ tpch_cn.region
+ where p_partkey = ps_partkey
+ and s_suppkey = ps_suppkey
+ and s_nationkey = n_nationkey
+ and n_regionkey = r_regionkey
+ and r_name = 'EUROPE')
+order by s_acctbal desc,
+ n_name,
+ s_name,
+ p_partkey
+limit 100;
+
+select min(ps_supplycost)
+from tpch_cn.partsupp,
+ tpch_cn.supplier,
+ tpch_cn.nation,
+ tpch_cn.region
+where s_suppkey = ps_suppkey
+ and s_nationkey = n_nationkey
+ and n_regionkey = r_regionkey
+ and r_name = 'EUROPE';
+
+select s_acctbal,
+ s_name,
+ n_name,
+ p_partkey,
+ p_mfgr,
+ s_address,
+ s_phone,
+ s_comment
+from
+ tpch_cn.partsupp,
+ tpch_cn.part,
+ tpch_cn.supplier,
+ tpch_cn.nation,
+ tpch_cn.region
+where p_partkey = ps_partkey
+ and s_suppkey = ps_suppkey
+ and p_size = 15
+ and p_type like '%BRASS'
+ and s_nationkey = n_nationkey
+ and n_regionkey = r_regionkey
+ and r_name = 'EUROPE'
+order by s_acctbal desc,
+ n_name,
+ s_name,
+ p_partkey
+limit 100;
+
+```
+
+
+### Query 3
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 3 : Shipping Priority Query / 运送优先级查询
+ * Return the first 10 selected rows
+ * 查询得到收入在前10位的尚未运送的订单。在指定的日期之前还没有运送的订单中具有最大收入的订单的运送优先级（订单按照收入的降序排序）和潜在的收入（潜在的收入为l_extendedprice * (1-l_discount)的和）。
+ * 带有分组、排序、聚集操作并存的三表查询操作。
+ */
+select l_orderkey,
+ sum(l_extendedprice * (1 - l_discount)) as revenue,
+ o_orderdate,
+ o_shippriority
+from tpch_cn.customer,
+ tpch_cn.orders,
+ tpch_cn.lineitem
+where c_mktsegment = 'BUILDING' -- SEGMENT is randomly selected within the list of values defined for Segments in Clause 4.2.2.13;
+ and c_custkey = o_custkey
+ and l_orderkey = o_orderkey
+ and o_orderdate < '1995-03-15' -- DATE is a randomly selected day within [1995-03-01 .. 1995-03-31].
+ and l_shipdate > '1995-03-15'
+group by l_orderkey,
+ o_orderdate,
+ o_shippriority
+order by revenue desc, o_orderdate
+limit 10;
+```
+
+
+```sql
+select l_orderkey,
+ sum(l_extendedprice * (1 - l_discount)) as revenue,
+ o_orderdate,
+ o_shippriority
+from tpch_cn.lineitem
+ join tpch_cn.orders on l_orderkey = o_orderkey
+ join tpch_cn.customer on c_custkey = o_custkey
+where c_mktsegment = 'BUILDING' -- SEGMENT is randomly selected within the list of values defined for Segments in Clause 4.2.2.13;
+ and o_orderdate < '1995-03-15' -- DATE is a randomly selected day within [1995-03-01 .. 1995-03-31].
+ and l_shipdate > '1995-03-15'
+group by l_orderkey,
+ o_orderdate,
+ o_shippriority
+order by revenue desc, o_orderdate
+limit 10;
+```
+
+
+### Query 4
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 4 : Order Priority Checking Query / 订单优先级查询
+ * 查询得到订单优先级统计值。计算给定的某三个月的订单的数量，在每个订单中至少有一行由顾客在它的提交日期之后收到。
+ * 带有分组、排序、聚集操作、子查询并存的单表查询操作。子查询是相关子查询。
+ */
+select o_orderpriority,
+ count(*) as order_count
+from tpch_cn.orders
+where o_orderdate >= '1993-07-01' -- DATE is the first day of a randomly selected month between the first month of 1993 and the 10th month of 1997.
+ and o_orderdate < DATE '1993-07-01' + interval '3' month
+ and exists ( -- exists 子查询
+ select *
+ from tpch_cn.lineitem
+ where l_orderkey = o_orderkey
+ and l_commitdate < l_receiptdate
+ )
+group by o_orderpriority
+order by o_orderpriority;
+```
+
+### Query 5
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 5 : Local Supplier Volume Query / 某地区供货商为公司带来的收入查询
+ * 通过某个地区零件供货商而获得的收入（收入按sum(l_extendedprice * (1 -l_discount))计算）统计信息。可用于决定在给定的区域是否需要建立一个当地分配中心
+ * 带有分组、排序、聚集操作、子查询并存的多表连接查询操作。
+ */
+select n_name,
+ sum(l_extendedprice * (1 - l_discount)) as revenue
+from tpch_cn.customer,
+ tpch_cn.orders,
+ tpch_cn.lineitem,
+ tpch_cn.supplier,
+ tpch_cn.nation,
+ tpch_cn.region
+where c_custkey = o_custkey
+ and l_orderkey = o_orderkey
+ and l_suppkey = s_suppkey
+ and c_nationkey = s_nationkey -- 这里形成一个环， Kylin 不支持这样的 Join Graph
+ and s_nationkey = n_nationkey
+ and n_regionkey = r_regionkey
+ and r_name = 'ASIA' -- REGION is randomly selected within the list of values defined for R_NAME in C;aise 4.2.3;
+ and o_orderdate >= date '1994-01-01' -- DATE is the first of January of a randomly selected year within [1993 .. 1997].
+ and o_orderdate < date '1994-01-01' + interval '1' year
+group by n_name
+order by revenue desc;
+```
+
+```sql
+select n1.n_name,
+ sum(l_extendedprice * (1 - l_discount)) as revenue
+from tpch_cn.lineitem
+ join tpch_cn.orders on l_orderkey = o_orderkey
+ join tpch_cn.customer on o_custkey = c_custkey
+ join tpch_cn.supplier on l_suppkey = s_suppkey
+ join tpch_cn.nation as n1 on s_nationkey = n1.n_nationkey
+ join tpch_cn.region as r1 on n1.n_regionkey = r1.r_regionkey
+ join tpch_cn.nation as n2 on c_nationkey = n2.n_nationkey
+ join tpch_cn.region as r2 on n2.n_regionkey = r2.r_regionkey
+where
+ r1.r_name = 'ASIA'
+ and o_orderdate >= date '1994-01-01'
+ and o_orderdate < date '1995-01-01'
+group by n1.n_name
+order by revenue desc;
+```
+
+
+### Query 6
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 6 : Forecasting Revenue Change Query / 预测收入变化查询
+ * 得到某一年中通过变换折扣带来的增量收入。这是典型的“what-if”判断，用来寻找增加收入的途径。
+ * 预测收入变化查询考虑了指定的一年中折扣在“DISCOUNT-0.01”和“DISCOUNT＋0.01”之间的已运送的所有订单，求解把l_quantity小于quantity的订单的折扣消除之后总收入增加的数量。
+ * 带有聚集操作的单表查询操作。查询语句使用了BETWEEN-AND操作符，有的数据库可以对BETWEEN-AND进行优化。
+ */
+select sum(l_extendedprice * l_discount) as revenue -- 潜在的收入增加量
+from tpch_cn.lineitem
+where l_shipdate >= '1994-01-01' -- DATE is the first of January of a randomly selected year within [1993 .. 1997];
+ and l_shipdate < DATE '1994-01-01' + interval '1' year
+ and l_discount between 0.06 - 0.01 and 0.06 + 0.01 -- DISCOUNT is randomly selected within [0.02 .. 0.09];
+ and l_quantity < 24 -- QUANTITY is randomly selected within [24 .. 25].
+;
+```
+
+```sql
+select sum(l_extendedprice * l_discount) as revenue -- 潜在的收入增加量
+from tpch_cn.lineitem
+where l_shipdate >= '1994-01-01' -- DATE is the first of January of a randomly selected year within [1993 .. 1997];
+ and l_shipdate < DATE '1995-01-01'
+ and l_discount between 0.06 - 0.01 and 0.06 + 0.01
+ and l_quantity < 24
+;
+```
+
+
+### Query 7(Skipped)
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 7 : Volume Shipping Query / 货运盈利情况查询
+ * 从供货商国家与销售商品的国家之间通过销售获利情况的查询。此查询确定在两国之间货运商品的量用以帮助重新谈判货运合同。
+ * 带有分组、排序、聚集、子查询操作并存的多表查询操作。子查询的父层查询不存在其他查询对象，是格式相对简单的子查询。
+ */
+select supp_nation,
+ cust_nation,
+ l_year,
+ sum(volume) as revenue
+from ( -- 子查询
+ select n1.n_name as supp_nation,
+ n2.n_name as cust_nation,
+ l_shipdate as l_year,
+ l_extendedprice * (1 - l_discount) as volume
+ from tpch_cn.supplier,
+ tpch_cn.lineitem,
+ tpch_cn.orders,
+ tpch_cn.customer,
+ tpch_cn.nation n1,
+ tpch_cn.nation n2
+ where s_suppkey = l_suppkey
+ and o_orderkey = l_orderkey
+ and c_custkey = o_custkey
+ and s_nationkey = n1.n_nationkey
+ and c_nationkey = n2.n_nationkey
+ and (
+ (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY') -- NATION1 is randomly selected within the list of values defined for N_NAME in Clause 4.2.3;
+ or
+ (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE') -- NATION2 is randomly selected within the list of values defined for N_NAME in Clause 4.2.3 and must be dif- ferent from the value selected for NATION1 in item 1 above.
+ )
+ and l_shipdate between '1995-01-01' and '1996-12-31'
+ ) as shipping
+group by supp_nation,
+ cust_nation,
+ l_year
+order by supp_nation,
+ cust_nation,
+ l_year;
+```
+
+### Query 8
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 8 : National Market Share Query / 国家市场份额查询
+ * 在过去的两年中一个给定零件类型在某国某地区市场份额的变化情况。
+ * 带有分组、排序、聚集、子查询操作并存的查询操作。子查询的父层查询不存在其他查询对象，是格式相对简单的子查询，但子查询自身是多表连接的查询。
+ */
+select o_year,
+ sum(case when nation = 'BRAZIL' then volume else 0 end) / sum(volume) -- NATION is randomly selected within the list of values defined for N_NAME in Clause 4.2.3;
+ as mkt_share
+from ( -- 子查询
+ select year(o_orderdate) as o_year,
+ l_extendedprice * (1 - l_discount) as volume,
+ n2.n_name as nation
+ from tpch_cn.part,
+ tpch_cn.supplier,
+ tpch_cn.lineitem,
+ tpch_cn.orders,
+ tpch_cn.customer,
+ tpch_cn.nation n1,
+ tpch_cn.nation n2,
+ tpch_cn.region
+ where p_partkey = l_partkey
+ and s_suppkey = l_suppkey
+ and l_orderkey = o_orderkey
+ and o_custkey = c_custkey
+ and c_nationkey = n1.n_nationkey
+ and n1.n_regionkey = r_regionkey
+ and r_name = 'AMERICA' -- REGION is the value defined in Clause 4.2.3 for R_NAME where R_REGIONKEY corresponds to N_REGIONKEY for the selected NATION in item 1 above;
+ and s_nationkey = n2.n_nationkey
+ and o_orderdate between '1995-01-01' and '1996-12-31'
+ and p_type = 'ECONOMY ANODIZED STEEL' -- TYPE is randomly selected within the list of 3-syllable strings defined for Types in Clause 4.2.2.13.
+ ) as all_nations
+group by o_year
+order by o_year;
+```
+
+
+### Query 9
+
+#### Original Version
+```sql
+/**
+ * TPC-H Query 9 : Product Type Profit Measure Query / 产品类型利润估量查询
+ * 查询每个国家每一年所有被定购的零件在一年中的总利润。
+ * 带有分组、排序、聚集、子查询操作并存的查询操作。子查询的父层查询不存在其他查询对象，是格式相对简单的子查询，但子查询自身是多表连接的查询。
+ * 子查询中使用了LIKE操作符，有的查询优化器不支持对LIKE操作符进行优化。
+ */
+select nation,
+ o_year,
+ sum(amount) as sum_profit
+from ( -- 子查询
+ select n_name as nation,
+ year(o_orderdate) as o_year,
+ l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount
+ from tpch_cn.part,
+ tpch_cn.supplier,
+ tpch_cn.lineitem,
+ tpch_cn.partsupp,
+ tpch_cn.orders,
+ tpch_cn.nation
+ where s_suppkey = l_suppkey
+ and ps_suppkey = l_suppkey
+ and ps_partkey = l_partkey
+ and p_partkey = l_partkey
+ and o_orderkey = l_orderkey
+ and s_nationkey = n_nationkey
+ and p_name like '%green%') as profit -- COLOR is randomly selected within the list of values defined for the generation of P_NAME in Clause 4.2.3.
+group by nation,
+ o_year
+order by nation,
+ o_year
+ desc;
+```
+
+#### Modified Version
+
+```sql
+ select n_name as nation,
+ year(o_orderdate) as o_year,
+ sum(l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity) as amount_profit
+ from tpch_cn.part,
+ tpch_cn.supplier,
+ tpch_cn.lineitem,
+ tpch_cn.partsupp,
+ tpch_cn.orders,
+ tpch_cn.nation
+ where s_suppkey = l_suppkey
+ and ps_suppkey = l_suppkey
+ and ps_partkey = l_partkey
+ and p_partkey = l_partkey
+ and o_orderkey = l_orderkey
+ and s_nationkey = n_nationkey
+ and p_name like '%green%'
+group by n_name, year(o_orderdate)
+order by n_name, year(o_orderdate);
+```
+
+
+### Query 10
+
+#### Original Version
+```sql
+/**
+ * TPC-H Query 10 : Returned Item Reporting Query / 货运存在问题的查询
+ * The Returned Item Reporting Query finds the top 20 customers
+ * 每个国家在某时刻起的三个月内货运存在问题的客户和造成的损失。
+ * 带有分组、排序、聚集操作并存的多表连接查询操作。
+ */
+select c_custkey,
+ c_name,
+ sum(l_extendedprice * (1 - l_discount)) as revenue,
+ c_acctbal,
+ n_name,
+ c_address,
+ c_phone,
+ c_comment
+from tpch_cn.customer,
+ tpch_cn.orders,
+ tpch_cn.lineitem,
+ tpch_cn.nation
+where c_custkey = o_custkey
+ and l_orderkey = o_orderkey
+ and o_orderdate >= date '1993-10-01' -- DATE is the first day of a randomly selected month from the second month of 1993 to the first month of 1995.
+ and o_orderdate < date '1993-10-01' + interval '3' month
+ and l_returnflag = 'R' -- 被退货
+ and c_nationkey = n_nationkey
+group by c_custkey,
+ c_name,
+ c_acctbal,
+ c_phone,
+ n_name,
+ c_address,
+ c_comment
+order by revenue desc
+limit 20;
+```
+
+
+### Query 11
+
+#### Original Version
+```sql
+/**
+ * TPC-H Query 11 : Important Stock Identification Query / 库存价值查询
+ * 查询库存中某个国家供应的零件的价值。
+ * 带有分组、排序、聚集、子查询操作并存的多表连接查询操作。子查询位于分组操作的HAVING条件中。
+ */
+select ps_partkey,
+ sum(ps_supplycost * ps_availqty) as value -- 聚集操作，商品的总价值
+from tpch_cn.partsupp,
+ tpch_cn.supplier,
+ tpch_cn.nation
+where ps_suppkey = s_suppkey
+ and s_nationkey = n_nationkey
+ and n_name = 'GERMANY' -- NATION is randomly selected within the list of values defined for N_NAME in Clause 4.2.3;
+group by ps_partkey
+having sum(ps_supplycost * ps_availqty) > ( -- 带有HAVING子句的分组操作
+ select sum(ps_supplycost * ps_availqty) * (0.0001 / 50) -- FRACTION is chosen as 0.0001 / SF.
+ from tpch_cn.partsupp,
+ tpch_cn.supplier,
+ tpch_cn.nation
+ where ps_suppkey = s_suppkey
+ and s_nationkey = n_nationkey
+ and n_name = 'GERMANY' -- 与父查询的 WHERE 条件一致
+)
+order by value desc;
+ ```
+
+### Query 12
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 12 : Shipping Modes and Order Priority Query / 货运模式和订单优先级查询
+ * 查询获得货运模式和订单优先级。可以帮助决策：选择便宜的货运模式是否会导致消费者更多的在合同日期之后收到货物，而对紧急优先命令产生负面影响。
+ * 带有分组、排序、聚集操作并存的两表连接查询操作。
+ */
+select l_shipmode,
+ sum(case
+ when o_orderpriority = '1-URGENT'
+ or o_orderpriority = '2-HIGH' then 1
+ else 0
+ end) as high_line_count,
+ sum(case
+ when o_orderpriority <> '1-URGENT'
+ and o_orderpriority <> '2-HIGH' then 1
+ else 0
+ end) as low_line_count
+from tpch_cn.orders,
+ tpch_cn.lineitem
+where o_orderkey = l_orderkey
+ and l_shipmode in ('MAIL', 'SHIP') -- SHIPMODE1 is randomly selected within the list of values defined for Modes in Clause 4.2.2.13; SHIPMODE2 is randomly selected within the list of values defined for Modes in Clause 4.2.2.13 and must be different from the value selected for SHIPMODE1 in item 1;
+ and l_commitdate < l_receiptdate
+ and l_shipdate < l_commitdate
+ and l_receiptdate >= '1994-01-01' -- DATE is the first of January of a randomly selected year within [1993 .. 1997].
+ and l_receiptdate < date '1995-01-01' + interval '1' year
+group by l_shipmode
+order by l_shipmode;
+```
+
+### Query 13
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 13 : Customer Distribution Query
+ * 查询获得消费者的订单数量，包括过去和现在都没有订单记录的消费者
+ * 带有分组、排序、聚集、子查询、左外连接操作并存的查询操作
+ */
+select c_count,
+ count(*) as custdist
+from
+( -- 子查询
+ select c_custkey, count(o_orderkey) as c_count
+ from tpch_cn.customer left outer join tpch_cn.orders -- 子查询中包括左外连接操作
+ on c_custkey = o_custkey and o_comment not like '%special%requests%'
+ -- WORD1 is randomly selected from 4 possible values: special, pending, unusual, express.
+ -- WORD2 is randomly selected from 4 possible values: packages, requests, accounts, deposits.
+ group by c_custkey
+) as c_orders
+group by
+ c_count
+order by
+ custdist desc,
+ c_count desc;
+```
+
+
+### Query 14
+
+#### Original Version
+
+ ```sql
+/**
+ * TPC-H Query 14 : Promotion Effect Query / 促销效果查询
+ * 查询获得某一个月的收入中有多大的百分比是来自促销零件。用以监视促销带来的市场反应。
+ */
+select 100.00 * sum(case
+ when p_type like 'PROMO%' -- 促销零件
+ then l_extendedprice * (1 - l_discount)
+ else 0
+ end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue
+from tpch_cn.lineitem,
+ tpch_cn.part
+where l_partkey = p_partkey
+ and l_shipdate >= '1995-09-01' -- DATE is the first day of a month randomly selected from a random year within [1993 .. 1997].
+ and l_shipdate < '1995-10-01'
+```
+
+
+### Query 15
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 15 : Top Supplier Query
+ * 查询获得某段时间内为总收入贡献最多的供货商（排名第一）的信息。可用以决定对哪些头等供货商给予奖励、给予更多订单、给予特别认证、给予鼓舞等激励。
+ * 带有分排序、聚集、聚集子查询操作并存的普通表与视图的连接操作。
+ */
+WITH revenue(supplier_no, total_revenue) as ( -- 复杂视图
+ SELECT l_suppkey,
+ SUM(l_extendedprice * (1 - l_discount))
+ FROM tpch_cn.lineitem
+ WHERE l_shipdate >= '1996-01-01' -- DATE is the first day of a randomly selected month between the first month of 1993 and the 10th month of 1997.
+ AND l_shipdate < date '1996-01-01' + interval '3' month
+ GROUP BY l_suppkey)
+SELECT s_suppkey,
+ s_name,
+ s_address,
+ s_phone,
+ total_revenue
+FROM tpch_cn.supplier,
+ revenue
+WHERE s_suppkey = supplier_no
+ AND total_revenue = ( -- 非关联子查询
+ SELECT MAX(total_revenue)
+ FROM revenue)
+ORDER BY s_suppkey;
+```
+
+
+```sql
+WITH revenue(supplier_no, total_revenue) as (
+ -- OLAPContext start
+ SELECT l_suppkey,
+ SUM(l_extendedprice * (1 - l_discount))
+ FROM tpch_cn.lineitem
+ WHERE l_shipdate >= '1996-01-01'
+ AND l_shipdate < '1996-04-01'
+ GROUP BY l_suppkey
+ -- OLAPContext end
+)
+SELECT s_suppkey,
+ s_name,
+ s_address,
+ s_phone,
+ total_revenue
+FROM tpch_cn.supplier,
+ revenue
+WHERE s_suppkey = supplier_no
+ AND total_revenue = (
+ -- OLAPContext start
+ SELECT MAX(total_revenue)
+ FROM revenue
+ -- OLAPContext end
+ )
+ORDER BY s_suppkey;
+```
+
+### Query 16
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 16 : Parts/Supplier Relationship Query / 零件/供货商关系查询
+ * 查询获得能够以指定的贡献条件供应零件的供货商数量。可用于决定在订单量大，任务紧急时，是否有充足的供货商。
+ * 带有分组、排序、聚集、去重、NOT IN 子查询操作并存的两表连接操作。
+ */
+select p_brand,
+ p_type,
+ p_size,
+ count(distinct ps_suppkey) as supplier_cnt
+from tpch_cn.partsupp,
+ tpch_cn.part
+where p_partkey = ps_partkey
+ and p_brand <> 'Brand#45' -- BRAND = Brand#MN where M and N are two single character strings representing two numbers randomly and independently selected within [1 .. 5];
+ and p_type not like 'MEDIUM POLISHED%' -- TYPE is made of the first 2 syllables of a string randomly selected within the list of 3-syllable strings defined for Types in Clause 4.2.2.13;
+ and p_size in (49, 14, 23, 45, 19, 3, 36, 9) -- SIZE is randomly selected as a set of eight different values within [1 .. 50]
+ and ps_suppkey not in (-- NOT IN子查询，消费者排除某些供货商
+ select s_suppkey
+ from tpch_cn.supplier
+ where s_comment like '%Customer%Complaints%')
+group by p_brand,
+ p_type,
+ p_size
+order by supplier_cnt desc,
+ p_brand,
+ p_type,
+ p_size;
+```
+
+### Query 17
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 17 : Small-Quantity-Order Revenue Query / 小订单收入查询
+ * 查询获得比平均供货量的百分之二十还低的小批量订单。对于指定品牌和指定包装类型的零件，决定在一个七年数据库的所有订单中这些订单零件的平均项目数量（过去的和未决的）。
+ * 如果这些零件中少于平均数20％的订单不再被接纳，那平均一年会损失多少呢？所以此查询可用于计算出如果没有没有小量订单，平均年收入将损失多少（因为大量商品的货运，将降低管理费用）。
+ * 带有聚集、聚集子查询操作并存的两表连接操作。
+ */
+select sum(l_extendedprice) / 7.0 as avg_yearly
+from tpch_cn.lineitem,
+ tpch_cn.part
+where p_partkey = l_partkey
+ and p_brand = 'Brand#23' -- BRAND = 'Brand#MN' where MN is a two character string representing two numbers randomly and independently selected within [1 .. 5];
+ and p_container = 'MED BOX' -- CONTAINER is randomly selected within the list of 2-syllable strings defined for Containers in Clause 4.2.2.13.
+ and l_quantity < (-- 关联子查询
+ select 0.2 * avg(l_quantity)
+ from tpch_cn.lineitem
+ where l_partkey = p_partkey);
+```
+
+### Query 18
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 18 : Large Volume Customer Query / 大订单顾客查询
+ * Return the first 100 selected rows
+ * 查询获得比指定供货量大的供货商信息。可用于决定在订单量大，任务紧急时，验证否有充足的供货商。
+ * 带有分组、排序、聚集、IN子查询操作并存的三表连接操作
+ */
+select c_name,
+ c_custkey,
+ o_orderkey,
+ o_orderdate,
+ o_totalprice,
+ sum(l_quantity)
+from tpch_cn.customer,
+ tpch_cn.orders,
+ tpch_cn.lineitem
+where o_orderkey in (-- 带有分组操作的IN子查询
+ select l_orderkey
+ from tpch_cn.lineitem
+ group by l_orderkey
+ having sum(l_quantity) > 312 -- QUANTITY is randomly selected within [312..315]
+ )
+ and c_custkey = o_custkey
+ and o_orderkey = l_orderkey
+group by c_name,
+ c_custkey,
+ o_orderkey,
+ o_orderdate,
+ o_totalprice
+order by o_totalprice desc,
+ o_orderdate
+limit 100;
+```
+
+### Query 19
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 19 : Discounted Revenue Query / 折扣收入查询
+ * BRAND1, BRAND2, BRAND3 = 'Brand#MN' where each MN is a two character string representing two numbers randomly and independently selected within [1 .. 5]
+ * 查询得到对一些空运或人工运输零件三个不同种类的所有订单的总折扣收入。零件的选择考虑特定品牌、包装和尺寸范围。本查询是用数据挖掘工具产生格式化代码的一个例子。
+ * 带有分组、排序、聚集、IN子查询操作并存的三表连接操作。
+ */
+select sum(l_extendedprice * (1 - l_discount)) as revenue
+from tpch_cn.lineitem,
+ tpch_cn.part
+where (
+ p_partkey = l_partkey
+ and p_brand = 'Brand#12'
+ and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG')
+ and l_quantity >= 1 and l_quantity <= 1 + 5 -- QUANTITY1 is randomly selected within [1..10].
+ and p_size between 1 and 5
+ and l_shipmode in ('AIR', 'AIR REG')
+ and l_shipinstruct = 'DELIVER IN PERSON'
+ )
+ or (
+ p_partkey = l_partkey
+ and p_brand = 'Brand#23'
+ and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK')
+ and l_quantity >= 10 and l_quantity <= 10 + 15 -- QUANTITY2 is randomly selected within [10..20].
+ and p_size between 1 and 10
+ and l_shipmode in ('AIR', 'AIR REG')
+ and l_shipinstruct = 'DELIVER IN PERSON'
+ )
+ or (
+ p_partkey = l_partkey
+ and p_brand = 'Brand#34'
+ and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG')
+ and l_quantity >= 20 and l_quantity <= 20 + 25 -- QUANTITY3 is randomly selected within [20..30].
+ and p_size between 1 and 15
+ and l_shipmode in ('AIR', 'AIR REG')
+ and l_shipinstruct = 'DELIVER IN PERSON'
+ );
+```
+
+### Query 20
+
+#### Original Version
+
+```sql
+/**
+ * TPC-H Query 20 : Potential Part Promotion Query / 供货商竞争力查询
+ * 查询确定在某一年内，找出指定国家的能对某一零件商品提供更有竞争力价格的供货货。所谓更有竞争力的供货商，是指那些零件有过剩的供货商，超过供或商在某一年中货运给定国的某一零件的50％则为过剩。
+ * 带有排序、聚集、IN子查询、普通子查询操作并存的两表连接操作。
+ */
+select s_name,
+ s_address
+from tpch_cn.supplier,
+ tpch_cn.nation
+where s_suppkey in ( -- 第一层子查询
+ select ps_suppkey
+ from tpch_cn.partsupp
+ where ps_partkey in ( -- 第二层子查询，非关联子查询
+ select p_partkey
+ from tpch_cn.part
+ where p_name like 'forest%'
+ )
+ and ps_availqty > ( -- 第二层子查询，关联子查询
+ select 0.5 * sum(l_quantity)
+ from tpch_cn.lineitem
+ where l_partkey = ps_partkey
+ and l_suppkey = ps_suppkey
+ and l_shipdate >= date('1994-01-01') -- DATE is the first of January of a randomly selected year within 1993..1997.
+ and l_shipdate < date('1994-01-01') + interval '1' year
+ )
+ ) -- 第一层子查询结束
+ and s_nationkey = n_nationkey
+ and n_name = 'CANADA' -- NATION is randomly selected within the list of values defined for N_NAME in Clause 4.2.3.
+order by s_name;
+```
+
+```sql
+
+with part_quantity as ( -- model1
+select
+ l_partkey,
+ l_suppkey,
+ 0.5 * sum(l_quantity) as half_sum_quantity
+from
+ tpch_cn.lineitem
+where l_shipdate >= '1994-01-01'
+ and l_shipdate < '1995-01-01'
+group by
+ l_partkey,
+ l_suppkey
+)
+
+with forest_suppilers as(
+select ps_suppkey
+from tpch_cn.partsupp
+ join part_quantity
+ on partsupp.ps_partkey = part_quantity.l_suppkey
+ and partsupp.ps_suppkey = part_quantity.l_suppkey
+ join tpch_cn.part
+ on partsupp.ps_partkey = part.p_partkey
+where ps_availqty > half_sum_quantity
+ and p_name like 'forest%'
+)
+select s_name, s_address
+from tpch_cn.supplier
+ join tpch_cn.nation on s_nationkey = n_nationkey
+ join forest_suppilers on supplier.s_suppkey= forest_suppilers.ps_suppkey
+where n_name = 'CANADA'
+;
+```
+
+### Query 21 : Suppliers Who Kept Orders Waiting Query
+
+```sql
+/**
+ * TPC-H Query 21 : Suppliers Who Kept Orders Waiting Query / 不能按时交货供货商查询
+ * Return the first 100 selected rows.
+ * 查询获得不能及时交货的供货商。
+ * 带有分组、排序、聚集、EXISTS子查询、NOT EXISTS子查询操作并存的四表连接操作。查询语句没有从语法上限制返回多少条元组，但是TPC-H标准规定，查询结果只返回前100行（通常依赖于应用程序实现）。
+ */
+select s_name,
+ count(*) as numwait
+from tpch_cn.supplier,
+ tpch_cn.lineitem l1,
+ tpch_cn.orders,
+ tpch_cn.nation
+where s_suppkey = l1.l_suppkey
+ and o_orderkey = l1.l_orderkey
+ and o_orderstatus = 'F'
+ and l1.l_receiptdate > l1.l_commitdate
+ and exists ( -- Exist 子查询
+ select *
+ from tpch_cn.lineitem l2
+ where l2.l_orderkey = l1.l_orderkey
+ and l2.l_suppkey <> l1.l_suppkey)
+ and not exists ( -- Not Exist 子查询
+ select *
+ from tpch_cn.lineitem l3
+ where l3.l_orderkey = l1.l_orderkey
+ and l3.l_suppkey <> l1.l_suppkey
+ and l3.l_receiptdate > l3.l_commitdate)
+ and s_nationkey = n_nationkey
+ and n_name = 'SAUDI ARABIA' -- NATION is randomly selected within the list of values defined for N_NAME in Clause 4.2.3.
+group by s_name
+order by numwait desc,
+ s_name;
+```
+
+
+### Query 22 : Global Sales Opportunity Query
+
+
+```sql
+/**
+* TPC-H Query 22 : Global Sales Opportunity Query / 全球销售机会查询
+* 查询获得消费者可能购买的地理分布。本查询计算在指定的国家，比平均水平更持肯定态度但还没下七年订单的消费者数量。能反应出普通消费者的的态度，即购买意向。
+* 带有分组、排序、聚集、EXISTS子查询、NOT EXISTS子查询操作并存的四表连接操作。
+*/
+select cntrycode,
+ count(*) as numcust,
+ sum(c_acctbal) as totacctbal
+from ( -- 第一层子查询
+ select substring(c_phone from 1 for 2) as cntrycode,
+ c_acctbal
+ from tpch_cn.customer
+ -- I1 ... I7 are randomly selected without repetition from the possible values for Country code as defined in Clause 4.2.2.9.
+ where substring(c_phone from 1 for 2) in ('13', '31', '23', '29', '30', '18', '17')
+ and c_acctbal > ( -- 第二层子查询
+ select avg(c_acctbal)
+ from tpch_cn.customer
+ where c_acctbal > 0.00
+ and substring(c_phone from 1 for 2)
+ in ('13', '31', '23', '29', '30', '18', '17'))
+ and not exists ( -- 第二层子查询
+ select *
+ from tpch_cn.orders
+ where o_custkey = c_custkey)
+ -- 第一层子查询结束
+ ) as custsale
+group by cntrycode
+order by cntrycode;
+```
\ No newline at end of file
diff --git a/dev-support/release-manager/tpch-benchmark/resources/TPCH-Schema.png b/dev-support/release-manager/tpch-benchmark/resources/TPCH-Schema.png
new file mode 100644
index 0000000000..5016cbbbec
Binary files /dev/null and b/dev-support/release-manager/tpch-benchmark/resources/TPCH-Schema.png differ
diff --git a/dev-support/release-manager/tpch-benchmark/scripts/create-all-tables.sql b/dev-support/release-manager/tpch-benchmark/scripts/create-all-tables.sql
new file mode 100644
index 0000000000..79553d34f1
--- /dev/null
+++ b/dev-support/release-manager/tpch-benchmark/scripts/create-all-tables.sql
@@ -0,0 +1,215 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-------------------------------------------------------------------------
+-------------------------------------------------------------------------
+--------------------------- Define & Formula ----------------------------
+-------------------------------------------------------------------------
+-------------------------------------------------------------------------
+
+-- 零售总价/l_extendedprice = L_QUANTITY * P_RETAILPRICE
+-- 销售收入/revenue = l_extendedprice * (1 - l_discount)
+-- 实际利润/profit = revenue - (ps_supplycost * l_quantity)
+-- 实际收费/charge = l_extendedprice*(1-l_discount)*(1+l_tax)
+-- L_LINESTATUS : "O" if L_SHIPDATE > CURRENTDATE "F" otherwise
+
+
+-------------------------------------------------------------------------
+-------------------------------------------------------------------------
+------------------------ Appendable Fact Table --------------------------
+-------------------------------------------------------------------------
+-------------------------------------------------------------------------
+
+CREATE DATABASE IF NOT EXISTS TPCH_CN;
+USE TPCH_CN;
+
+DROP TABLE IF EXISTS lineitem;
+DROP TABLE IF EXISTS orders;
+DROP TABLE IF EXISTS customer;
+DROP TABLE IF EXISTS partsupp;
+DROP TABLE IF EXISTS part;
+DROP TABLE IF EXISTS supplier;
+DROP TABLE IF EXISTS nation;
+DROP TABLE IF EXISTS region;
+
+
+CREATE EXTERNAL TABLE lineitem(
+ l_orderkey bigint ,
+ l_partkey bigint ,
+ l_suppkey bigint ,
+ l_linenumber int ,
+ l_quantity DECIMAL(10,2),
+ l_extendedprice DECIMAL(14,2),
+ l_discount DECIMAL(4,3) ,
+ l_tax DECIMAL(4,3),
+ l_returnflag string,
+ l_linestatus string,
+ l_shipdate date,
+ l_commitdate date,
+ l_receiptdate date,
+ l_shipinstruct string ,
+ l_shipmode string ,
+ l_comment string
+)
+COMMENT 'Single-Row-of-Order-6M'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+ WITH SERDEPROPERTIES ('path'='hdfs://kylin5-machine:9000/user/data/tpch/lineitem')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION 'hdfs://kylin5-machine:9000/user/data/tpch/lineitem'
+;
+
+
+CREATE EXTERNAL TABLE orders(
+ o_orderkey bigint,
+ o_custkey bigint ,
+ o_orderstatus string ,
+ o_totalprice DECIMAL(21,3) ,
+ o_orderdate date ,
+ o_orderpriority string ,
+ o_clerk string ,
+ o_shippriority int ,
+ o_comment string
+)
+COMMENT 'Order-1_5M'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+ WITH SERDEPROPERTIES ('path'='hdfs://kylin5-machine:9000/user/data/tpch/orders')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION 'hdfs://kylin5-machine:9000/user/data/tpch/orders'
+;
+
+
+-------------------------------------------------------------------------
+-------------------------------------------------------------------------
+---------------------------- Small Fact Table ---------------------------
+-------------------------------------------------------------------------
+-------------------------------------------------------------------------
+
+CREATE EXTERNAL TABLE customer(
+ c_custkey bigint,
+ c_name string,
+ c_address string,
+ c_nationkey bigint,
+ c_phone string ,
+ c_acctbal decimal(15,3),
+ c_mktsegment string,
+ c_comment string
+)
+COMMENT 'Customers-150k'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+ WITH SERDEPROPERTIES ('path'='hdfs://kylin5-machine:9000/user/data/tpch/customer')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION 'hdfs://kylin5-machine:9000/user/data/tpch/customer'
+;
+
+
+
+CREATE EXTERNAL TABLE partsupp(
+ ps_partkey bigint,
+ ps_suppkey bigint,
+ ps_availqty int,
+ ps_supplycost DECIMAL(15,3),
+ ps_comment string
+)
+COMMENT 'Part-Supplier-Relation-800k'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+ WITH SERDEPROPERTIES ('path'='hdfs://kylin5-machine:9000/user/data/tpch/partsupp')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION 'hdfs://kylin5-machine:9000/user/data/tpch/partsupp'
+;
+
+
+CREATE EXTERNAL TABLE part(
+ p_partkey bigint,
+ p_name string,
+ p_mfgr string,
+ p_brand string,
+ p_type string,
+ p_size int ,
+ p_container string,
+ p_retailprice DECIMAL(15,3),
+ p_comment string
+)
+COMMENT 'Part-200k'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+ WITH SERDEPROPERTIES ('path'='hdfs://kylin5-machine:9000/user/data/tpch/part')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+ OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+ 'hdfs://kylin5-machine:9000/user/data/tpch/part'
+;
+
+
+CREATE EXTERNAL TABLE supplier(
+ s_suppkey bigint,
+ s_name string,
+ s_address string,
+ s_nationkey bigint,
+ s_phone string,
+ s_acctbal DECIMAL(15,3),
+ s_comment string)
+COMMENT 'Supplier-10k'
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+ WITH SERDEPROPERTIES (
+ 'path'='hdfs://kylin5-machine:9000/user/data/tpch/supplier')
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+ OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+ 'hdfs://kylin5-machine:9000/user/data/tpch/supplier'
+;
+
+
+-------------------------------------------------------------------------
+-------------------------------------------------------------------------
+---------------------------- Dimension Table ----------------------------
+-------------------------------------------------------------------------
+-------------------------------------------------------------------------
+
+CREATE EXTERNAL TABLE nation(
+ n_nationkey bigint,
+ n_name string,
+ n_regionkey bigint,
+ n_comment string
+)
+COMMENT 'Nation-25'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+ WITH SERDEPROPERTIES ('path'='hdfs://kylin5-machine:9000/user/data/tpch/nation')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION 'hdfs://kylin5-machine:9000/user/data/tpch/nation'
+;
+
+CREATE EXTERNAL TABLE region(
+ r_regionkey bigint,
+ r_name string ,
+ r_comment string
+)
+COMMENT 'Region-5'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+ WITH SERDEPROPERTIES ('path'='hdfs://kylin5-machine:9000/user/data/tpch/region')
+STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION 'hdfs://kylin5-machine:9000/user/data/tpch/region'
+;
\ No newline at end of file
diff --git a/dev-support/release-manager/tpch-benchmark/scripts/transform-to-parquet.scala b/dev-support/release-manager/tpch-benchmark/scripts/transform-to-parquet.scala
new file mode 100644
index 0000000000..7d17a77671
--- /dev/null
+++ b/dev-support/release-manager/tpch-benchmark/scripts/transform-to-parquet.scala
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// export HADOOP_CONF_DIR=/opt/hadoop-3.2.1/etc/hadoop
+// bin/spark-shell --executor-cores 1 --num-executors 8 --master yarn --conf spark.sql.files.maxPartitionBytes=30m
+
+spark.read.format("orc").load("/user/hive/warehouse/tpch_flat_orc_2.db/lineitem").registerTempTable("lineitem")
+spark.read.format("orc").load("/user/hive/warehouse/tpch_flat_orc_2.db/orders").registerTempTable("orders")
+spark.read.format("orc").load("/user/hive/warehouse/tpch_flat_orc_2.db/customer").registerTempTable("customer")
+spark.read.format("orc").load("/user/hive/warehouse/tpch_flat_orc_2.db/partsupp").registerTempTable("partsupp")
+spark.read.format("orc").load("/user/hive/warehouse/tpch_flat_orc_2.db/part").registerTempTable("part")
+spark.read.format("orc").load("/user/hive/warehouse/tpch_flat_orc_2.db/supplier").registerTempTable("supplier")
+spark.read.format("orc").load("/user/hive/warehouse/tpch_flat_orc_2.db/nation").registerTempTable("nation")
+spark.read.format("orc").load("/user/hive/warehouse/tpch_flat_orc_2.db/region").registerTempTable("region")
+
+var lt = spark.sql("""
+select
+ l_orderkey,
+ l_partkey,
+ l_suppkey,
+ l_linenumber,
+ CAST(l_quantity AS DECIMAL(10,2)) as l_quantity,
+ CAST(l_extendedprice AS DECIMAL(14,2)) as l_extendedprice,
+ CAST(l_discount AS DECIMAL(4,3)) as l_discount,
+ CAST(l_tax AS DECIMAL(4,3)) as l_tax,
+ l_returnflag,
+ l_linestatus,
+ to_date(l_shipdate) as l_shipdate,
+ to_date(l_commitdate) as l_commitdate,
+ to_date(l_receiptdate) as l_receiptdate,
+ l_shipinstruct,
+ l_shipmode,
+ l_comment
+from lineitem
+""").cache()
+
+lt.write.format("parquet").save("/user/hive/warehouse/xxyu/lineitem")
+
+lt = spark.sql("""
+select
+ o_orderkey,
+ o_custkey,
+ o_orderstatus,
+ CAST(o_totalprice AS DECIMAL(21,3)) as o_totalprice,
+ to_date(o_orderdate) as o_orderdate,
+ o_orderpriority,
+ o_clerk,
+ o_shippriority,
+ o_comment
+from orders
+""").cache()
+
+lt.write.format("parquet").save("/user/hive/warehouse/xxyu/orders")
+
+
+lt = spark.sql("""
+select
+ c_custkey,
+ c_name,
+ c_address,
+ c_nationkey,
+ c_phone,
+ CAST(c_acctbal AS DECIMAL(15,3)) as c_acctbal,
+ c_mktsegment,
+ c_comment
+from customer
+""").cache()
+
+lt.write.format("parquet").save("/user/hive/warehouse/xxyu/customer")
+
+lt = spark.sql("""
+select
+ ps_partkey,
+ ps_suppkey,
+ ps_availqty,
+ CAST(ps_supplycost AS DECIMAL(15,3)) as ps_supplycost,
+ ps_comment
+from partsupp
+""").cache()
+
+lt.write.format("parquet").save("/user/hive/warehouse/xxyu/partsupp")
+
+lt = spark.sql("""
+select
+ p_partkey,
+ p_name,
+ p_mfgr,
+ p_brand,
+ p_type,
+ p_size,
+ p_container,
+ CAST(p_retailprice AS DECIMAL(15,3)) as p_retailprice,
+ p_comment
+from part
+""").cache()
+
+lt.write.format("parquet").save("/user/hive/warehouse/xxyu/part")
+
+
+lt = spark.sql("""
+select
+ s_suppkey,
+ s_name,
+ s_address,
+ s_nationkey,
+ s_phone,
+ CAST(s_acctbal AS DECIMAL(15,3)) as s_acctbal,
+ s_comment
+from supplier
+""").cache()
+
+lt.write.format("parquet").save("/user/hive/warehouse/xxyu/supplier")
+
+
+lt = spark.sql("""
+select
+ *
+from nation
+""").cache()
+
+lt.write.format("parquet").save("/user/hive/warehouse/xxyu/nation")
+
+
+lt = spark.sql("""
+select
+ *
+from region
+""").cache()
+
+lt.write.format("parquet").save("/user/hive/warehouse/xxyu/region")
\ No newline at end of file
diff --git a/dev-support/release-manager/tpch-benchmark/scripts/update-hive-column-comment.sql b/dev-support/release-manager/tpch-benchmark/scripts/update-hive-column-comment.sql
new file mode 100644
index 0000000000..832c3cb3c3
--- /dev/null
+++ b/dev-support/release-manager/tpch-benchmark/scripts/update-hive-column-comment.sql
@@ -0,0 +1,98 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- How to execute this file.
+-- mysql -u root -p123456 hive < this-file
+-- mysql -u root -p123456 hive -e "update COLUMNS_V2 set COMMENT = '该条目所属的订单编号,外键,联合主键' where COLUMN_NAME = 'l_orderkey' ;"
+
+alter table COLUMNS_V2 modify column COMMENT varchar(256) character set utf8mb4;
+
+-- lineitem
+update COLUMNS_V2 set COMMENT = '该条目所属的订单编号,外键,联合主键' where COLUMN_NAME = 'l_orderkey' ;
+update COLUMNS_V2 set COMMENT = '销售零件编号' where COLUMN_NAME = 'l_partkey' ;
+update COLUMNS_V2 set COMMENT = '销售零件所属供应商编号' where COLUMN_NAME = 'l_suppkey' ;
+update COLUMNS_V2 set COMMENT = '该条目在所属订单的序号:1-7,联合主键' where COLUMN_NAME = 'l_linenumber' ;
+update COLUMNS_V2 set COMMENT = '销售数量' where COLUMN_NAME = 'l_quantity' ;
+update COLUMNS_V2 set COMMENT = '零售价格:L_QUANTITY * P_RETAILPRICE' where COLUMN_NAME = 'l_extendedprice' ;
+update COLUMNS_V2 set COMMENT = '促销折扣，百分比' where COLUMN_NAME = 'l_discount' ;
+update COLUMNS_V2 set COMMENT = '税率，税由顾客支付，百分比' where COLUMN_NAME = 'l_tax' ;
+update COLUMNS_V2 set COMMENT = '退货标志位:R,A,F' where COLUMN_NAME = 'l_returnflag' ;
+update COLUMNS_V2 set COMMENT = '条目发送状态:O,F' where COLUMN_NAME = 'l_linestatus' ;
+update COLUMNS_V2 set COMMENT = '配送日期' where COLUMN_NAME = 'l_shipdate' ;
+update COLUMNS_V2 set COMMENT = '承诺送达日期' where COLUMN_NAME = 'l_commitdate' ;
+update COLUMNS_V2 set COMMENT = '实际送达日期' where COLUMN_NAME = 'l_receiptdate' ;
+update COLUMNS_V2 set COMMENT = '配送方式:DELIVER IN PERSON,COLLECT COD,NONE,TAKE BACK RETURN' where COLUMN_NAME = 'l_shipinstruct' ;
+update COLUMNS_V2 set COMMENT = '配送交通方式:REG AIR,AIR,RAIL,SHIP,TRUCK,MAIL' where COLUMN_NAME = 'l_shipmode' ;
+update COLUMNS_V2 set COMMENT = '订单备注' where COLUMN_NAME = 'l_comment';
+
+
+-- orders
+update COLUMNS_V2 set COMMENT = '订单编号，主键' where COLUMN_NAME = 'o_orderkey' ;
+update COLUMNS_V2 set COMMENT = '顾客编号，外键' where COLUMN_NAME = 'o_custkey' ;
+update COLUMNS_V2 set COMMENT = '订单状态，取决于l_linestatus，可选值:O,F,P' where COLUMN_NAME = 'o_orderstatus' ;
+update COLUMNS_V2 set COMMENT = '条目零售价格总合:sum(L_EXTENDEDPRICE*(1+L_TAX)*(1-L_DISCOUNT))' where COLUMN_NAME = 'o_totalprice' ;
+update COLUMNS_V2 set COMMENT = '订单日期' where COLUMN_NAME = 'o_orderdate' ;
+update COLUMNS_V2 set COMMENT = '订单优先级' where COLUMN_NAME = 'o_orderpriority' ;
+update COLUMNS_V2 set COMMENT = '职员编号:Clerk#000000000' where COLUMN_NAME = 'o_clerk' ;
+update COLUMNS_V2 set COMMENT = '配送优先级' where COLUMN_NAME = 'o_shippriority' ;
+update COLUMNS_V2 set COMMENT = '订单备注' where COLUMN_NAME = 'o_comment';
+
+-- customer
+update COLUMNS_V2 set COMMENT = '顾客编号，主键' where COLUMN_NAME = 'c_custkey' ;
+update COLUMNS_V2 set COMMENT = '顾客称呼' where COLUMN_NAME = 'c_name' ;
+update COLUMNS_V2 set COMMENT = '顾客地址' where COLUMN_NAME = 'c_address' ;
+update COLUMNS_V2 set COMMENT = '顾客国家，外键' where COLUMN_NAME = 'c_nationkey' ;
+update COLUMNS_V2 set COMMENT = '顾客手机号码' where COLUMN_NAME = 'c_phone' ;
+update COLUMNS_V2 set COMMENT = '顾客余额，范围:[-999.99, 9999.99]' where COLUMN_NAME = 'c_acctbal' ;
+update COLUMNS_V2 set COMMENT = '可选值:AUTOMOBILE,BUILDING,FURNITURE,MACHINERY,HOUSEHOLD' where COLUMN_NAME = 'c_mktsegment' ;
+update COLUMNS_V2 set COMMENT = '_' where COLUMN_NAME = 'c_comment';
+
+
+-- PS
+update COLUMNS_V2 set COMMENT = '零件编号,外键,联合主键' where COLUMN_NAME = 'ps_partkey' ;
+update COLUMNS_V2 set COMMENT = '供应商编号,外键,联合主键' where COLUMN_NAME = 'ps_suppkey' ;
+update COLUMNS_V2 set COMMENT = '该供应商持有该零件的剩余数量:[1,9999]' where COLUMN_NAME = 'ps_availqty' ;
+update COLUMNS_V2 set COMMENT = '进货成本/批发价:[1.00,1000.00]' where COLUMN_NAME = 'ps_supplycost' ;
+update COLUMNS_V2 set COMMENT = '_' where COLUMN_NAME = 'ps_comment';
+
+
+-- part
+update COLUMNS_V2 set COMMENT = '零件编号,主键' where COLUMN_NAME = 'p_partkey' ;
+update COLUMNS_V2 set COMMENT = '零件名称' where COLUMN_NAME = 'p_name' ;
+update COLUMNS_V2 set COMMENT = '制造商:Manufacturer#[1-5]' where COLUMN_NAME = 'p_mfgr' ;
+update COLUMNS_V2 set COMMENT = '品牌:Brand#[1-5][1-5]' where COLUMN_NAME = 'p_brand' ;
+update COLUMNS_V2 set COMMENT = '零件类型，三个单词的组合' where COLUMN_NAME = 'p_type' ;
+update COLUMNS_V2 set COMMENT = '零件尺寸:[1,50]' where COLUMN_NAME = 'p_size' ;
+update COLUMNS_V2 set COMMENT = '包装方式，两个单词的组合' where COLUMN_NAME = 'p_container' ;
+update COLUMNS_V2 set COMMENT = '零件零售价' where COLUMN_NAME = 'p_retailprice' ;
+
+-- suppiler
+update COLUMNS_V2 set COMMENT = '供应商编号,主键' where COLUMN_NAME = 's_suppkey' ;
+update COLUMNS_V2 set COMMENT = '供应商名称' where COLUMN_NAME = 's_name' ;
+update COLUMNS_V2 set COMMENT = '供应商地址' where COLUMN_NAME = 's_address' ;
+update COLUMNS_V2 set COMMENT = '外键' where COLUMN_NAME = 's_nationkey' ;
+update COLUMNS_V2 set COMMENT = '供应商手机号' where COLUMN_NAME = 's_phone' ;
+update COLUMNS_V2 set COMMENT = '供应商账户余额' where COLUMN_NAME = 's_acctbal' ;
+
+-- nation
+update COLUMNS_V2 set COMMENT = '主键,0-24' where COLUMN_NAME = 'n_nationkey' ;
+update COLUMNS_V2 set COMMENT = '25个国家:ALGERIA,ARGENTINA,BRAZIL,CANADA,EGYPT,etc' where COLUMN_NAME = 'n_name' ;
+
+-- region
+update COLUMNS_V2 set COMMENT = '外键' where COLUMN_NAME = 'n_regionkey' ;
+update COLUMNS_V2 set COMMENT = '主键,0-4' where COLUMN_NAME = 'r_regionkey' ;
diff --git a/kystudio/src/components/layout/layout_left_right_top.vue b/kystudio/src/components/layout/layout_left_right_top.vue
index 343cdb400b..9247310a00 100644
--- a/kystudio/src/components/layout/layout_left_right_top.vue
+++ b/kystudio/src/components/layout/layout_left_right_top.vue
@@ -74,6 +74,7 @@
<el-button type="primary" text iconr="el-ksd-icon-arrow_down_22"><span class="limit-user-name">{{currentUserInfo && currentUserInfo.username}}</span></el-button>
<el-dropdown-menu slot="dropdown">
<div class="user-name">{{ currentUserInfo && currentUserInfo.username }}</div>
+ <div class="user-name"><a href="https://kylin.apache.org/5.0/docs/quickstart/tutorial" target="_blank">Tutorial</a></div>
<el-dropdown-item :disabled="!isTestingSecurityProfile" command="setting">{{$t('kylinLang.common.changePassword')}}</el-dropdown-item>
<el-dropdown-item command="loginout">{{$t('kylinLang.common.logout')}}</el-dropdown-item>
</el-dropdown-menu>
@@ -846,7 +847,7 @@ export default class LayoutLeftRightTop extends Vue {
}
.user-name {
padding: 0 8px 8px 8px;
- line-height: 1;
+ line-height: 2;
border-bottom: 1px solid @line-split-color;
box-sizing: border-box;
overflow: hidden;
diff --git a/kystudio/src/components/query/insight.vue b/kystudio/src/components/query/insight.vue
index 8e87909107..0a6a419d2b 100644
--- a/kystudio/src/components/query/insight.vue
+++ b/kystudio/src/components/query/insight.vue
@@ -559,7 +559,7 @@ export default class NewQuery extends Vue {
}
}
.submit-tips {
- float: right;
+ float: left;
font-size: 12px;
line-height: 18px;
vertical-align: middle;
diff --git a/kystudio/src/components/query/query_tab.vue b/kystudio/src/components/query/query_tab.vue
index b5c4f6f183..d9cd4eb117 100644
--- a/kystudio/src/components/query/query_tab.vue
+++ b/kystudio/src/components/query/query_tab.vue
@@ -30,7 +30,9 @@
</div>
<div class="submit-tips" v-if="isWorkspace">
<i class="el-icon-ksd-info ksd-fs-12" ></i>
- Control / Command + Enter = <span>{{!isLoading ? $t('runQuery') : $t('stopQuery')}}</span></div>
+ Control / Command + Enter = <span>{{!isLoading ? $t('runQuery') : $t('stopQuery')}}</span>
+ Tip : Dryrun is an experimental feature for user to create proper model. <br>To enable/disable dry run, please set 'kylin.query.dryrun-enabled=true/false' in Setting -> Advanced Settings -> Custom Project Configuration.
+ </div>
</div>
<div v-show="isLoading" class="ksd-center ksd-mt-10">
<el-progress type="circle" :percentage="percent"></el-progress>
@@ -104,7 +106,7 @@ import { kylinConfirm, handleSuccess, handleError } from '../../util/business'
viewLogs: 'View Logs',
htraceTips: 'Please make sure Zipkin server is properly deployed according to the manual of performance diagnose package.',
queryError: 'The query fails.',
- queryTimeOut: 'The request timeout, please check the network situation and Kylin service instance status. If the resource group is turned on, please make sure that the project is bound to the query resource group and there are available query instances',
+ queryTimeOut: 'The request timeout, please check the network situation and Kylin service instance status.',
queryTimeOutInCloud: 'The request timeout, please check the network situation and Kylin 5 service instance status.'
}
}
diff --git a/kystudio/src/components/setting/SettingBasic/SettingBasic.vue b/kystudio/src/components/setting/SettingBasic/SettingBasic.vue
index 47b52460bc..a5b89fbdd6 100644
--- a/kystudio/src/components/setting/SettingBasic/SettingBasic.vue
+++ b/kystudio/src/components/setting/SettingBasic/SettingBasic.vue
@@ -12,8 +12,8 @@
<div class="setting-value fixed">{{project.alias || project.project}}</div>
</div>
<div class="setting-item">
- <div class="setting-label font-medium">Package Info</div>
- <div class="setting-value fixed">kylin.version {{project.package_version}}, {{project.package_timestamp}}, commit: {{project.git_commit}}</div>
+ <div class="setting-label font-medium">Developer Info</div>
+ <div class="setting-value fixed"> {{project.package_version}} [{{project.package_timestamp}}, {{project.git_commit}}]</div>
</div>
<div class="setting-item clearfix">
<div class="setting-label font-medium">{{$t('description')}}</div>
diff --git a/pom.xml b/pom.xml
index b8d26f8d14..534e8bcca6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@
<!-- Spark versions -->
<delta.version>2.0.2</delta.version>
- <spark.version>3.2.0-kylin-4.6.11.0</spark.version>
+ <spark.version>3.2.0-kylin-4.6.9.0</spark.version>
<roaring.version>0.9.2-kylin-r4</roaring.version>
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java
index 18c2a939c7..9675b130ee 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java
@@ -57,6 +57,27 @@ public class MetricsConfig {
@EventListener(ApplicationReadyEvent.class)
public void registerMetrics() {
+ if (KylinConfig.getInstanceFromEnv().isPrometheusMetricsEnabled()) {
+ log.info("Register prometheus global metrics... ");
+ MetricsRegistry.registerGlobalPrometheusMetrics();
+
+ METRICS_SCHEDULED_EXECUTOR.scheduleAtFixedRate(() -> {
+ Set<String> allProjects = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).listAllProjects()
+ .stream().map(ProjectInstance::getName).collect(Collectors.toSet());
+
+ MetricsRegistry.refreshProjectLongRunningJobs(KylinConfig.getInstanceFromEnv(), allProjects);
+ Sets.SetView<String> newProjects = Sets.difference(allProjects, allControlledProjects);
+ for (String newProject : newProjects) {
+ log.info("Register prometheus metrics for project {}", newProject);
+ MetricsRegistry.registerProjectPrometheusMetrics(KylinConfig.getInstanceFromEnv(), newProject);
+ }
+
+ allControlledProjects.clear();
+ allControlledProjects.addAll(allProjects);
+
+ }, 0, 1, TimeUnit.MINUTES);
+ }
+
if (!KapConfig.getInstanceFromEnv().isMonitorEnabled()) {
return;
}
@@ -70,10 +91,6 @@ public class MetricsConfig {
log.info("Register host metrics...");
MetricsRegistry.registerHostMetrics(host);
- if (KylinConfig.getInstanceFromEnv().isPrometheusMetricsEnabled()) {
- log.info("Register prometheus global metrics... ");
- MetricsRegistry.registerGlobalPrometheusMetrics();
- }
METRICS_SCHEDULED_EXECUTOR.scheduleAtFixedRate(() -> {
Set<String> allProjects = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).listAllProjects()
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/MetricsRegistry.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/MetricsRegistry.java
index 8431dd5e37..dfc05c8960 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/MetricsRegistry.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/MetricsRegistry.java
@@ -199,22 +199,27 @@ public class MetricsRegistry {
return count;
}).tags(MetricsTag.STATE.getVal(), state, MetricsTag.POOL.getVal(), "dbcp2",
MetricsTag.TYPE.getVal(), driver)
+ .description("Number of Metastore(RDBMS) connections")
.strongReference(true).register(meterRegistry));
}
Gauge.builder(PrometheusMetrics.SPARDER_UP.getValue(), () -> SparderEnv.isSparkAvailable() ? 1 : 0)
+ .description("Health status of spark context(query engine)")
.strongReference(true).register(meterRegistry);
Gauge.builder(PrometheusMetrics.SPARK_TASKS.getValue(), LoadCounter.getInstance(),
e -> SparderEnv.isSparkAvailable() ? e.getPendingTaskCount() : 0)
.tags(MetricsTag.STATE.getVal(), MetricsTag.PENDING.getVal()).strongReference(true)
+ .description("Number of pending spark tasks of query engine")
.register(meterRegistry);
Gauge.builder(PrometheusMetrics.SPARK_TASKS.getValue(), LoadCounter.getInstance(),
e -> SparderEnv.isSparkAvailable() ? e.getRunningTaskCount() : 0)
.tags(MetricsTag.STATE.getVal(), MetricsTag.RUNNING.getVal()).strongReference(true)
+ .description("Number of running spark tasks of query engine")
.register(meterRegistry);
Gauge.builder(PrometheusMetrics.SPARK_TASK_UTILIZATION.getValue(), LoadCounter.getInstance(),
e -> SparderEnv.isSparkAvailable() ? e.getRunningTaskCount() * 1.0 / e.getSlotCount() : 0)
+ .description("Ratio of spark cores utilization.")
.strongReference(true).register(meterRegistry);
}
@@ -229,7 +234,8 @@ public class MetricsRegistry {
() -> Objects.isNull(scheduler.getContext()) ? 0
: scheduler.getContext().getRunningJobs().values().stream()
.filter(job -> ExecutableState.RUNNING.equals(job.getOutput().getState())).count())
- .tags(projectTag).tags(MetricsTag.STATE.getVal(), MetricsTag.RUNNING.getVal()).register(meterRegistry);
+ .tags(projectTag).tags(MetricsTag.STATE.getVal(), MetricsTag.RUNNING.getVal())
+ .description("Number of spark job by build engine").register(meterRegistry);
for (double runningTimeoutHour : RUNNING_JOB_TIMEOUT_HOUR) {
Gauge.builder(PrometheusMetrics.JOB_LONG_RUNNING.getValue(),
@@ -237,6 +243,7 @@ public class MetricsRegistry {
.getOrDefault(runningTimeoutHour, 0L))
.tags(projectTag).tags(MetricsTag.STATE.getVal(), MetricsTag.RUNNING.getVal(),
MetricsTag.TIMEOUT.getVal(), runningTimeoutHour + "h")
+ .description("Number of spark job by query engine")
.register(meterRegistry);
}
@@ -246,6 +253,7 @@ public class MetricsRegistry {
.getOrDefault(waitTimeoutMin, 0L))
.tags(projectTag).tags(MetricsTag.STATE.getVal(), MetricsTag.WAITING.getVal(),
MetricsTag.TIMEOUT.getVal(), waitTimeoutMin + "m")
+ .description("Number of spark job by build engine which ")
.register(meterRegistry);
}
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryMetricsListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryMetricsListener.java
index 7e6d064e04..4ebebf225e 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryMetricsListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryMetricsListener.java
@@ -92,19 +92,21 @@ public class QueryMetricsListener {
MetricsTag.HIT_SECOND_STORAGE.getVal(), queryMetric.isSecondStorage() + "")
.distributionStatisticExpiry(Duration.ofDays(1))
.sla(KylinConfig.getInstanceFromEnv().getMetricsQuerySlaSeconds())
+ .description("Query duration")
.register(meterRegistry)
.record(queryMetric.getQueryDuration() * 1.0 / 1000);
if (queryMetric.isSucceed()) {
DistributionSummary.builder(PrometheusMetrics.QUERY_RESULT_ROWS.getValue()).tags(projectTag)
+ .description("Number of rows returned by query")
.distributionStatisticExpiry(Duration.ofDays(1)).register(meterRegistry)
.record(queryMetric.getResultRowCount());
- Counter.builder(PrometheusMetrics.QUERY_JOBS.getValue()).tags(projectTag).register(meterRegistry)
+ Counter.builder(PrometheusMetrics.QUERY_JOBS.getValue()).tags(projectTag).description("Number of spark job by query engine").register(meterRegistry)
.increment(queryMetric.getQueryJobCount());
- Counter.builder(PrometheusMetrics.QUERY_STAGES.getValue()).tags(projectTag).register(meterRegistry)
+ Counter.builder(PrometheusMetrics.QUERY_STAGES.getValue()).tags(projectTag).description("Number of spark stage by query engine").register(meterRegistry)
.increment(queryMetric.getQueryStageCount());
- Counter.builder(PrometheusMetrics.QUERY_TASKS.getValue()).tags(projectTag).register(meterRegistry)
+ Counter.builder(PrometheusMetrics.QUERY_TASKS.getValue()).tags(projectTag).description("Number of spark task by query engine").register(meterRegistry)
.increment(queryMetric.getQueryTaskCount());
}
@@ -115,6 +117,7 @@ public class QueryMetricsListener {
.map(e -> modelManager.getDataModelDesc(e.getModelId()).getAlias())
.collect(Collectors.joining(",")),
MetricsTag.PROJECT.getVal(), queryMetric.getProjectName())
+ .description("Total scanned bytes by query")
.distributionStatisticExpiry(Duration.ofDays(1)).publishPercentiles(new double[] { 0.8, 0.9 })
.register(meterRegistry).record(queryMetric.getTotalScanBytes());
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
index 73cacef7af..a526310691 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
@@ -517,7 +517,6 @@ public class KapConfig {
}
/**
- * Newten
*/
public String getCuboidSpanningTree() {
return config.getOptional("kylin.cube.cuboid-spanning-tree",
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3614898c9a..2908f05571 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1245,7 +1245,6 @@ public abstract class KylinConfigBase implements Serializable {
Map<Integer, String> r = Maps.newLinkedHashMap();
// ref constants in ISourceAware
- // these sources no longer exists in Newten
r.put(1, "org.apache.kylin.source.kafka.NSparkKafkaSource");
r.put(8, "org.apache.kylin.source.jdbc.JdbcSource");
r.put(9, "org.apache.kylin.engine.spark.source.NSparkDataSource");
@@ -1895,6 +1894,10 @@ public abstract class KylinConfigBase implements Serializable {
return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-bytes", String.valueOf(1024 * 1024)));
}
+ public boolean isQueryDryRunEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.dryrun-enabled", FALSE));
+ }
+
public boolean isQueryCacheEnabled() {
return Boolean.parseBoolean(this.getOptional("kylin.query.cache-enabled", TRUE));
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 7422bff2d7..36d1594724 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -184,7 +184,7 @@ public class KylinVersion implements Comparable {
if (gitInfo != null) {
return gitInfo;
}
- gitInfo = new Pair<>("N/A", "N/A");
+ gitInfo = new Pair<>("-", "-");
try {
File commitFile = new File(KylinConfig.getKylinHome(), COMMIT_SHA1_v15);
if (!commitFile.exists()) {
@@ -192,9 +192,9 @@ public class KylinVersion implements Comparable {
}
List<String> lines = FileUtils.readLines(commitFile, Charset.defaultCharset());
for (String line : lines) {
- if (line.contains("@") && gitInfo.getFirst().equals("N/A")) {
+ if (line.contains("@") && gitInfo.getFirst().equals("-")) {
gitInfo.setFirst(line);
- } else if (line.contains("timestamp") && gitInfo.getSecond().equals("N/A")) {
+ } else if (line.contains("timestamp") && gitInfo.getSecond().equals("-")) {
String[] words = line.split(":");
if (words.length == 2) {
gitInfo.setSecond(words[1]);
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index cda5ec2c33..d364fd2526 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -145,6 +145,23 @@ public class QueryContext implements Closeable {
@Setter
private boolean enhancedAggPushDown;
+
+ /**
+ * For debug purpose, will show RelNode
+ * when dryRun is enabled
+ */
+ @Getter
+ @Setter
+ private String lastUsedRelNode;
+
+ /**
+ * For debug purpose, when dry run mode is enabled, the query
+ * will be not executed, and debug message of this query will
+ * show in the Insight page.
+ */
+ @Getter
+ private boolean dryRun = false;
+
private QueryContext() {
// use QueryContext.current() instead
queryId = RandomUtil.randomUUIDStr();
@@ -189,6 +206,10 @@ public class QueryContext implements Closeable {
recordMillis = current;
}
+ public void setDryRun(boolean flag) {
+ dryRun = flag;
+ }
+
@Override
public void close() {
reset();
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/DryRunSucceedException.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/DryRunSucceedException.java
new file mode 100644
index 0000000000..1497ba201f
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/DryRunSucceedException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.exception;
+
+/**
+ * When dryrun is enabled, the query will not
+ * be executed by Spark, instead print some
+ * debug message.
+ */
+public class DryRunSucceedException extends KylinRuntimeException {
+ String sparkPlan = "";
+
+ public DryRunSucceedException(String msg, String planStr){
+ super(msg);
+ sparkPlan = planStr;
+ }
+
+ public String plan(){
+ return sparkPlan;
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NAggregationGroup.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NAggregationGroup.java
index 59188bea3f..cd4ff72cf9 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NAggregationGroup.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NAggregationGroup.java
@@ -47,9 +47,6 @@ import org.apache.kylin.guava30.shaded.common.math.LongMath;
import lombok.Getter;
import lombok.Setter;
-/**
- * to compatible with legacy aggregation group in pre-newten
- */
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class NAggregationGroup implements Serializable {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
index d04e0bacb8..22336c3bc4 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
@@ -689,6 +689,7 @@ public class IndexPlan extends RootPersistentEntity implements Serializable, IEn
markIndexesToBeDeleted(indexPlanId, toBeDeletedSet, Collections.emptyMap());
}
+ // white index : exclude the indexes generated by Agg
public void markWhiteIndexToBeDelete(String indexPlanId, final Set<Long> layoutIds,
Map<Long, Boolean> secondStorageLayoutStatus) {
Preconditions.checkNotNull(indexPlanId);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflow.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflow.java
index 525c45959c..9c7542492a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflow.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflow.java
@@ -284,7 +284,7 @@ public class NDataflow extends RootPersistentEntity implements Serializable, IRe
@Override
public boolean isOnline() {
- return getStatus() == RealizationStatusEnum.ONLINE;
+ return getStatus() == RealizationStatusEnum.ONLINE || config.isQueryDryRunEnabled();
}
@Override
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NativeQueryRealization.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NativeQueryRealization.java
index d12bd1019c..588724faf8 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NativeQueryRealization.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NativeQueryRealization.java
@@ -52,6 +52,8 @@ public class NativeQueryRealization implements Serializable {
@Setter
private NDataModelAclParams aclParams;
+ private int cxtId = -1;
+
public NativeQueryRealization(String modelId, String modelAlias, Long layoutId, String indexType,
boolean isPartialMatchModel, List<String> snapshots) {
this.modelId = modelId;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 04c5a4dd82..f24089bb55 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -51,7 +51,7 @@ public class SQLDigest {
// group by
public List<TblColRef> groupbyColumns;
- public Set<TblColRef> subqueryJoinParticipants; // FIXME: can we add subqueryJoinParticipants to allColumns/groupbyCols at OLAPContext? from dong@newten
+ public Set<TblColRef> subqueryJoinParticipants; // FIXME: can we add subqueryJoinParticipants to allColumns/groupbyCols at OLAPContext?
// aggregation
public Set<TblColRef> metricColumns;
diff --git a/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
index de789e0020..dfb086eae2 100644
--- a/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
+++ b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
@@ -96,6 +96,7 @@ public class MetricsGroup {
public static boolean counterInc(MetricsName name, MetricsCategory category, String entity,
Map<String, String> tags, long increments) {
+ if(!KapConfig.getInstanceFromEnv().isMonitorEnabled()) return true;
if (increments < 0) {
return false;
}
@@ -120,6 +121,7 @@ public class MetricsGroup {
public static boolean histogramUpdate(MetricsName name, MetricsCategory category, String entity,
Map<String, String> tags, long updateTo) {
+ if(!KapConfig.getInstanceFromEnv().isMonitorEnabled()) return true;
if (updateTo < 0) {
return false;
}
@@ -442,6 +444,7 @@ public class MetricsGroup {
public static final Map<String, MetricsObject> cacheInfluxMetricStatusMap = new ConcurrentHashMap<>();
public static long tryRestoreCounter(String fieldName, String category, String entity, Map<String, String> tags) {
+ if(!KapConfig.getInstanceFromEnv().isMonitorEnabled()) return 0L;
try {
MetricsObject metricsObject = new MetricsObject(fieldName, category, entity, tags);
if (MetricsName.QUERY.getVal().equals(fieldName)) {
@@ -500,6 +503,7 @@ public class MetricsGroup {
}
private static long tryRestoreExceptionCounter(String fieldName, String category, String entity, Map<String, String> tags) {
+ if(!KapConfig.getInstanceFromEnv().isMonitorEnabled()) return 0L;
MetricsObject metricsObject = new MetricsObject(fieldName, category, entity, tags);
if (KylinConfig.getInstanceFromEnv().isDevOrUT()) {
cacheInfluxMetricStatusMap.get(metricsObject.toString()).setInitStatus(true);
diff --git a/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java
index 30bd8f03f1..8a3428351e 100644
--- a/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java
+++ b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java
@@ -57,6 +57,7 @@ public class InfluxMetricsTest extends NLocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
NLocalFileMetadataTestCase.staticCreateTestMetadata();
+ getTestConfig().setProperty("kylin.monitor.enabled", "true");
}
@Test
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
index 8b9204836f..d08d0d073d 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
@@ -36,6 +36,7 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexSlot;
import org.apache.calcite.sql.fun.SqlCountAggFunction;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.util.RexUtils;
import org.apache.kylin.util.CalciteSystemProperty;
@@ -113,6 +114,9 @@ public class ContextUtil {
if (Boolean.TRUE.equals(CalciteSystemProperty.DEBUG.value()) && logger.isDebugEnabled()) {
logger.debug("{} :{}{}", msg, System.getProperty("line.separator"), RelOptUtil.toString(relNode));
}
+ if (QueryContext.current().isDryRun() && msg.contains("FIRST ROUND")) {
+ QueryContext.current().setLastUsedRelNode(RelOptUtil.toString(relNode));
+ }
}
private static boolean derivedFromSameContext(Collection<Integer> indexOfInputCols, RelNode currentNode,
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index e2d00d77b3..eadb698c06 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -65,6 +65,7 @@ import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.query.routing.RealizationCheck;
import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.storage.StorageContext;
+import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -297,6 +298,7 @@ public class OLAPContext {
String modelId, String modelAlias, List<String> snapshots) {
val realization = new NativeQueryRealization(modelId, modelAlias, ctx.storageContext.getLayoutId(),
realizationType, ctx.storageContext.isPartialMatchModel(), snapshots);
+ realization.setCxtId(ctx.id);
realization.setSecondStorage(
QueryContext.current().getSecondStorageUsageMap().getOrDefault(realization.getLayoutId(), false));
realization.setRecommendSecondStorage(
@@ -527,6 +529,43 @@ public class OLAPContext {
+ ", filterColumns=" + filterColumns + '}';
}
+ public final static String SEP = System.getProperty("line.separator");
+ public final static String INDENT = " ";
+
+ public final static String olapContextFormat = SEP
+ + "{"
+ + SEP + INDENT + "\"Fact Table\" = \"%s\","
+ + SEP + INDENT + "\"Dimension Tables\" = [%s],"
+ + SEP + INDENT + "\"Recommend Dimension(Group by)\" = [%s],"
+ + SEP + INDENT + "\"Recommend Dimension(Filter cond)\" = [%s],"
+ + SEP + INDENT + "\"Measures\" = [%s],"
+ + SEP + "}"
+ + SEP;
+
+ public String tipsForUser() {
+ Set<String> allTables = allTableScans.stream().map(OLAPTableScan::getTableName).collect(Collectors.toSet());
+ if (!allTables.isEmpty() && firstTableScan != null) {
+ allTables.remove(firstTableScan.getTableName());
+ return String.format(olapContextFormat,
+ firstTableScan.getTableName(),
+ Strings.join(allTables.stream().map(c -> "\"" + c + "\"").iterator(), ','),
+ Strings.join(groupByColumns.stream().map(c -> "\"" + c.getColumnWithTableAndSchema() + "\"").iterator(), ','),
+ Strings.join(filterColumns.stream().map(c -> "\"" + c.getColumnWithTableAndSchema() + "\"").iterator(), ','),
+ Strings.join(aggregations.stream().map(c -> "\"" + c.getFullExpression() + "\"").iterator(), ',')
+ );
+ } else {
+ return "empty";
+ }
+ }
+
+ public String toHumanReadString() {
+ String r = realization == null ? "not matched" : realization.getCanonicalName();
+ return "{id = " + id
+ + ", model = " + r
+ + ", fact table = " + (firstTableScan != null ? firstTableScan.getTableName() : "?")
+ + "}";
+ }
+
public void matchJoinWithFilterTransformation() {
Set<TableRef> leftOrInnerTables = getNotNullTables();
if (CollectionUtils.isEmpty(leftOrInnerTables)) {
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
index 6ac0c81af5..aec64d8794 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
@@ -20,16 +20,21 @@ package org.apache.kylin.query.routing;
import java.util.Comparator;
import java.util.List;
+import java.util.Locale;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Ordering;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.project.NProjectManager;
import lombok.Getter;
+import org.apache.kylin.metadata.realization.CapabilityResult;
+import org.apache.kylin.metadata.realization.IRealization;
public class QueryRouter {
@@ -87,6 +92,29 @@ public class QueryRouter {
public Strategy(KylinConfig config) {
+ // add debug rule
+ if (config.isQueryDryRunEnabled() && QueryContext.current().getModelPriorities().length > 0) {
+ // This will remove all candidate which not listed in SQL Hint 'MODEL_PRIORITY', for debug
+ rules.add(new PruningRule() {
+ @Override
+ public void apply(Candidate candidate) {
+ Set<String> models = Sets.newHashSet(QueryContext.current().getModelPriorities());
+ boolean matched = false;
+ for (IRealization r : candidate.getRealization().getRealizations()) {
+ NDataflow df = (NDataflow) r;
+ if (models.contains(df.getModelAlias().toUpperCase(Locale.ROOT))) {
+ matched = true;
+ }
+ }
+ if (!matched) {
+ CapabilityResult capability = new CapabilityResult();
+ capability.setCapable(false);
+ candidate.setCapability(capability);
+ }
+ }
+ });
+ }
+
// add all rules
rules.add(SEGMENT_PRUNING);
rules.add(PARTITION_PRUNING);
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index eb82e60b35..7ae0238f04 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -203,15 +203,15 @@ public class RealizationChooser {
Multimap<NDataModel, IRealization> modelMap = makeOrderedModelMap(context);
checkNoRealizationFound(context, modelMap);
- logger.info("Context join graph: {}", context.getJoinsGraph());
+ logger.info("Context join graph: {}, {}", context.toHumanReadString(), context.getJoinsGraph());
// Step 2.1 try to exactly match model
List<Candidate> candidates = trySelectCandidates(context, modelMap, false, false);
// Step 2.2 try to partial match model
+ String project = context.olapSchema.getProjectName();
+ KylinConfig projectConfig = NProjectManager.getProjectConfig(project);
if (CollectionUtils.isEmpty(candidates)) {
- String project = context.olapSchema.getProjectName();
- KylinConfig projectConfig = NProjectManager.getProjectConfig(project);
boolean partialMatch = projectConfig.isQueryMatchPartialInnerJoinModel();
boolean nonEquiPartialMatch = projectConfig.partialMatchNonEquiJoins();
if (partialMatch || nonEquiPartialMatch) {
@@ -223,7 +223,8 @@ public class RealizationChooser {
if (candidates.isEmpty()) {
checkNoRealizationWithStreaming(context);
RelAggPushDownUtil.registerUnmatchedJoinDigest(context.getTopNode());
- throw new NoRealizationFoundException("No realization found for " + toErrorMsg(context));
+ String msg = projectConfig.isQueryDryRunEnabled() ? helpfulMessageForUser(context) : toErrorMsg(context);
+ throw new NoRealizationFoundException("No realization found for " + msg);
}
// Step 3. find the lowest-cost candidate
@@ -263,7 +264,9 @@ public class RealizationChooser {
}
checkNoRealizationWithStreaming(context);
RelAggPushDownUtil.registerUnmatchedJoinDigest(context.getTopNode());
- throw new NoRealizationFoundException("No model found for " + toErrorMsg(context));
+ String msg = NProjectManager.getProjectConfig(context.olapSchema.getProjectName()).isQueryDryRunEnabled()
+ ? helpfulMessageForUser(context) : toErrorMsg(context);
+ throw new NoRealizationFoundException("No realization found for " + msg);
}
private static List<Candidate> trySelectCandidates(OLAPContext context, Multimap<NDataModel, IRealization> modelMap,
@@ -341,8 +344,8 @@ public class RealizationChooser {
}
Candidate candidate = new Candidate(realization, olapContext, matchedJoinGraphAliasMap);
- logger.info("Find candidates by table {} and project={} : {}", olapContext.firstTableScan.getTableName(),
- olapContext.olapSchema.getProjectName(), candidate);
+ logger.info("Find candidates for {}, by table {} and project={} : {}", olapContext.id, olapContext.firstTableScan.getTableName(),
+ olapContext.olapSchema.getProjectName(), candidate.getRealization().getModel().getAlias());
QueryRouter.applyRules(candidate);
@@ -350,8 +353,7 @@ public class RealizationChooser {
return null;
}
- logger.info("The realizations remaining: {}, and the final chosen one for current olap context {} is {}",
- candidate.realization.getCanonicalName(), olapContext.id, candidate.realization.getCanonicalName());
+ logger.info("Chosen model for current olap context {} is {}", olapContext.id, candidate.realization.getCanonicalName());
return candidate;
}
@@ -396,6 +398,7 @@ public class RealizationChooser {
}
private static boolean hasReadySegments(NDataModel model) {
+ if (QueryContext.current().isDryRun()) return true;
val dataflow = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), model.getProject())
.getDataflow(model.getUuid());
if (model.isFusionModel()) {
@@ -664,6 +667,21 @@ public class RealizationChooser {
return buf.toString();
}
+ private static String helpfulMessageForUser(OLAPContext ctx) {
+ StringBuilder buf = new StringBuilder(ctx.toHumanReadString());
+ buf.append(System.getProperty("line.separator"));
+ if(ctx.getJoinsGraph() != null) {
+ buf.append(" Join graph : ").append(ctx.getJoinsGraph().toString()).append(System.getProperty("line.separator"));
+ }
+ buf.append(" Incapable message : ");
+ for (List<RealizationCheck.IncapableReason> reasons : ctx.realizationCheck.getModelIncapableReasons().values()) {
+ for (RealizationCheck.IncapableReason reason : reasons) {
+ buf.append(reason).append(", ");
+ }
+ }
+ return buf.toString();
+ }
+
public static Map<String, String> matchJoins(NDataModel model, OLAPContext ctx, boolean partialMatchInnerJoin,
boolean partialMatchNonEquiJoin) {
Map<String, String> matchedAliasMap = Maps.newHashMap();
@@ -700,8 +718,10 @@ public class RealizationChooser {
partialMatchNonEquiJoin);
logger.info("Match result for match join with join match optimization mode is: {}", matched);
}
- logger.debug("Context join graph missed model {}, model join graph {}", model, model.getJoinsGraph());
- logger.debug("Missed match nodes - Context {}, Model {}",
+ logger.debug("Context [{}] join graph missed model {}, model join graph {}, unmatched model join graph {}, unmatched olap join graph {}",
+ ctx.id,
+ model.getAlias(),
+ model.getJoinsGraph(),
ctx.getJoinsGraph().unmatched(model.getJoinsGraph()),
model.getJoinsGraph().unmatched(ctx.getJoinsGraph()));
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 5654c9a928..b6d2630a96 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -195,7 +195,7 @@ public class QueryUtil {
queryParams.getOffset());
queryParams.setSql(massagedSql);
massagedSql = transformSql(queryParams);
- QueryContext.current().record("massage");
+ QueryContext.current().record("end_massage_sql");
return massagedSql;
}
diff --git a/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java b/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java
index 2cbff5994c..173881e61a 100644
--- a/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java
+++ b/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java
@@ -36,8 +36,6 @@ import io.swagger.annotations.ApiOperation;
/**
* Backward capable API for : /kylin/api/tables_and_columns
- **
- * TODO ODBC should support Newten API: /kylin/api/query/tables_and_columns
*
* @author yifanzhang
*
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryMetricsListenerTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryMetricsListenerTest.java
index f3ffdddb84..e82dae7528 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryMetricsListenerTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryMetricsListenerTest.java
@@ -65,6 +65,7 @@ public class QueryMetricsListenerTest extends NLocalFileMetadataTestCase {
@Before
public void setup() {
createTestMetadata();
+ getTestConfig().setProperty("kylin.monitor.enabled", "true");
queryMetricsListener = Mockito.spy(new QueryMetricsListener());
queryService = Mockito.spy(new QueryService());
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
index 80295c22e8..31cd18b979 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
@@ -51,6 +51,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryTrace;
import org.apache.kylin.common.ReadFsSwitch;
+import org.apache.kylin.common.exception.DryRunSucceedException;
import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -169,6 +170,10 @@ public class QueryExec {
magicDirts(sql);
QueryContext queryContext = QueryContext.current();
try {
+ if (kylinConfig.isQueryDryRunEnabled()) {
+ logger.trace("Dry run Mode.");
+ QueryContext.current().setDryRun(true);
+ }
beforeQuery();
QueryContext.currentTrace().startSpan(QueryTrace.SQL_PARSE_AND_OPTIMIZE);
RelRoot relRoot = sqlConverter.convertSqlToRelNode(sql);
@@ -508,6 +513,82 @@ public class QueryExec {
}
private SQLException newSqlException(String sql, String msg, Throwable e) {
- return new SQLException("Error while executing SQL \"" + sql + "\": " + msg, e);
+ String diagnosticStr = "";
+ String plan = " not exists";
+ boolean dryRun = QueryContext.current().isDryRun();
+ if(e instanceof DryRunSucceedException){
+ DryRunSucceedException d = (DryRunSucceedException)e;
+ plan = d.plan();
+ }
+ if(dryRun) {
+ StringBuilder diagnosticInfo = new StringBuilder(System.getProperty("line.separator"));
+ diagnosticInfo.append("<-------------------- Dry Run Info -------------------->");
+
+ diagnosticInfo.append(SEP)
+ .append("1. Last Exception :")
+ .append(SEP).append(" ");
+ diagnosticInfo.append(msg).append(SEP);
+
+ diagnosticInfo.append(SEP)
+ .append("2. RelNode(with ctx id) :")
+ .append(SEP).append(" ");
+ diagnosticInfo.append(QueryContext.current().getLastUsedRelNode());
+
+ diagnosticInfo.append(SEP)
+ .append("3. OLAPContext(s) and matched model(s) :");
+ if(OLAPContext.getThreadLocalContexts() != null) {
+ String olapMatchInfo =
+ OLAPContext.getNativeRealizations().stream()
+ .map(r -> String.format(" Ctx=%d, \tMatched=%s, \tIndexType=%s, \tLayoutId=%d",
+ r.getCxtId(), r.getModelAlias(), r.getIndexType(), r.getLayoutId()))
+ .collect(Collectors.joining(SEP));
+ if (olapMatchInfo.length() >= 10) {
+ diagnosticInfo.append(SEP)
+ .append(olapMatchInfo)
+ .append(SEP);
+ }
+
+ for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) {
+ diagnosticInfo.append(SEP);
+ diagnosticInfo.append(" Ctx=").append(ctx.id);
+ if(ctx.realization == null) {
+ diagnosticInfo.append(" is not matched by any model/snapshot, expected ")
+ .append(ctx.tipsForUser());
+ } else {
+ diagnosticInfo.append(" is matched by ").append(ctx.realization.getCanonicalName())
+ .append(", expected ").append(ctx.tipsForUser());;
+ }
+// diagnosticInfo.append(", verbose text:").append(ctx);
+ }
+ diagnosticInfo.append(SEP);
+ }
+
+ diagnosticInfo.append(SEP)
+ .append("4. SQL Text :")
+ .append(SEP);
+ diagnosticInfo.append(sql)
+ .append(SEP);
+
+ diagnosticInfo.append(SEP)
+ .append("5. Physical plan :")
+ .append(SEP);
+ diagnosticInfo.append(plan).append(SEP);
+
+ diagnosticInfo.append(SEP)
+ .append("<-------------------- Dry Run Info -------------------->");
+ diagnosticInfo.append(SEP)
+ .append(DRY_RUN_TIP);
+ diagnosticStr = diagnosticInfo.toString();
+ return new SQLException(diagnosticStr, e);
+ } else {
+ return new SQLException("Error while executing SQL \"" + sql + "\": " + msg, e);
+ }
}
+
+ public final static String SEP = System.getProperty("line.separator");
+ public final static String DRY_RUN_TIP = SEP
+ + "Tip : Dryrun is a experimental feature for user to create proper model."
+ + SEP + "To enable/disable dry run, please set 'kylin.query.dryrun-enabled=true/false'"
+ + SEP + "in Setting -> Advanced Settings -> Custom Project Configuration."
+ + SEP;
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/SQLConverter.java b/src/query/src/main/java/org/apache/kylin/query/engine/SQLConverter.java
index e80bdfbb83..ff6140543c 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/SQLConverter.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/SQLConverter.java
@@ -103,7 +103,7 @@ public class SQLConverter {
*/
public RelRoot convertSqlToRelNode(String sql) throws SqlParseException {
SqlNode sqlNode = parseSQL(sql);
- QueryContext.current().record("end calcite parse sql");
+ QueryContext.current().record("end_calcite_parse_sql");
return convertToRelNode(sqlNode);
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java b/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
index 5d4e133447..fc6734dfcc 100644
--- a/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
+++ b/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
@@ -82,7 +82,7 @@ public class QueryContextCutter {
} catch (NoRealizationFoundException | NoStreamingRealizationFoundException e) {
if (isReCutBanned && e instanceof NoStreamingRealizationFoundException) {
checkStreamingTableWithAutoModeling();
- } else if (isReCutBanned || e instanceof NoStreamingRealizationFoundException) {
+ } else if (isReCutBanned || e instanceof NoStreamingRealizationFoundException || QueryContext.current().isDryRun()) {
throw e;
}
reCutStrategy.tryCutToSmallerContexts(root, e);
@@ -115,7 +115,7 @@ public class QueryContextCutter {
List<OLAPContext> contexts = ContextUtil.listContextsHavingScan();
contexts.forEach(olapContext -> {
olapContext.setHasSelected(true);
- log.info("Context for realization matching: {}", olapContext);
+ log.info("Context to be match {}, {}", olapContext.id, olapContext);
});
long selectLayoutStartTime = System.currentTimeMillis();
@@ -126,7 +126,7 @@ public class QueryContextCutter {
}
log.info("select layout candidate for {} olapContext cost {} ms", contexts.size(),
System.currentTimeMillis() - selectLayoutStartTime);
- QueryContext.current().record("end select realization");
+ QueryContext.current().record("end_select_realization");
return contexts;
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/util/QueryModelPriorities.java b/src/query/src/main/java/org/apache/kylin/query/util/QueryModelPriorities.java
index 22233d0cd6..e196ff7d54 100644
--- a/src/query/src/main/java/org/apache/kylin/query/util/QueryModelPriorities.java
+++ b/src/query/src/main/java/org/apache/kylin/query/util/QueryModelPriorities.java
@@ -43,7 +43,6 @@ public class QueryModelPriorities {
if (models.length > 0) {
return models;
}
- // for backward compatibility with KE3
return loadCubePriorityFromComment(sql);
}
@@ -60,7 +59,6 @@ public class QueryModelPriorities {
return modelHints;
}
- // for backward compatibility with KE3
private static final Pattern CUBE_PRIORITY_PATTERN = Pattern
.compile("(?<=--(\\s){0,2}CubePriority\\().*(?=\\)(\\s)*[\r\n])");
diff --git a/src/server/src/main/resources/log4j2.xml b/src/server/src/main/resources/log4j2.xml
index d3b9e32a6e..7fa5985383 100644
--- a/src/server/src/main/resources/log4j2.xml
+++ b/src/server/src/main/resources/log4j2.xml
@@ -19,6 +19,7 @@
<Appenders>
<Console name="stdout" target="SYSTEM_OUT">
<PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p %X{request.project}[%t] %c{2} : %mask{%m}%n"/>
+<!-- <PatternLayout pattern="%-5p %d{HH:mm:ss,SSS} %X{request.project}[%t] ==== %l %c{2}%n %mask{%m}%n%n"/>-->
</Console>
</Appenders>
<Loggers>
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparkEngine.java b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparkEngine.java
index 13d94113fb..2d5d510894 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparkEngine.java
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparkEngine.java
@@ -25,6 +25,7 @@ import org.apache.calcite.rel.RelNode;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryTrace;
+import org.apache.kylin.common.exception.DryRunSucceedException;
import org.apache.kylin.query.engine.exec.ExecuteResult;
import org.apache.kylin.query.engine.exec.sparder.QueryEngine;
import org.apache.kylin.query.mask.QueryResultMasks;
@@ -63,6 +64,10 @@ public class SparkEngine implements QueryEngine {
@Override
public ExecuteResult computeToIterable(DataContext dataContext, RelNode relNode) {
Dataset<Row> sparkPlan = QueryResultMasks.maskResult(toSparkPlan(dataContext, relNode));
+ if (QueryContext.current().isDryRun()) {
+ throw new DryRunSucceedException("DryRun succeed. Query is stopped due to DryRun enabled.",
+ sparkPlan.queryExecution().executedPlan().toString());
+ }
log.info("SPARK LOGICAL PLAN {}", sparkPlan.queryExecution().logical());
if (KapConfig.getInstanceFromEnv().isOnlyPlanInSparkEngine()) {
return ResultPlan.completeResultForMdx(sparkPlan, relNode.getRowType());