You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/09/06 15:11:55 UTC
[kylin] 01/01: KYLIN-4660 Fix QueryMetricsTest
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a31a2b11d9334f9817663a32e0a084f957ad63f4
Author: XiaoxiangYu <hi...@126.com>
AuthorDate: Sun Sep 6 23:09:13 2020 +0800
KYLIN-4660 Fix QueryMetricsTest
- QueryMetricsTest will not start a sparder context.
- Skip some module because I want to make sure UT pass at 30 mins.
---
.travis.yml | 4 +-
KEYS | 58 ++++++++++++++++++++++
build/conf/kylin-tools-log4j.properties | 3 +-
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++
.../java/org/apache/kylin/common/KylinVersion.java | 2 +-
.../src/main/resources/kylin-defaults.properties | 6 +--
core-dictionary/pom.xml | 14 +++++-
.../build}/conf/kylin-tools-log4j.properties | 11 ++--
kylin-spark-project/kylin-spark-test/pom.xml | 18 +++++--
.../kylin/rest/init/InitialSparderContext.java | 5 +-
10 files changed, 103 insertions(+), 22 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 6e53d12..cbb9598 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -39,7 +39,9 @@ before_script:
&& mvn -version
- echo "MAVEN_OPTS='-Xms1024m -Xmx3072m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m'" > ~/.mavenrc
- sed -i 's/log4j.logger.org.apache.kylin=DEBUG/log4j.logger.org.apache.kylin=WARN/g' build/conf/kylin-server-log4j.properties
- - sed -i 's/log4j.logger.org.apache.kylin=DEBUG/log4j.logger.org.apache.kylin=WARN/g' build/conf/kylin-tools-log4j.properties
+ - sed -i 's/log4j.logger.org.apache.kylin=INFO/log4j.logger.org.apache.kylin=WARN/g' build/conf/kylin-tools-log4j.properties
+ - sed -i 's/log4j.logger.org.apache.kylin=INFO/log4j.logger.org.apache.kylin=WARN/g' kylin-spark-project/build/conf/kylin-tools-log4j.properties
+
script:
- mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent test coveralls:report -e
diff --git a/KEYS b/KEYS
index 8c0fa0b..00ab8d0 100644
--- a/KEYS
+++ b/KEYS
@@ -478,3 +478,61 @@ mUdVyu44Yf7c3XnrZL0F7mwzFmpgeTWGOTzqfO5YSeHr5x8gizsBVADaoB1uPjPz
kzyYjjy3XzKvLlVIHuqksvDyibrlU1StUmNS64lfugMY1RfLERUpRdV3
=DvSV
-----END PGP PUBLIC KEY BLOCK-----
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+mQINBF9UsH4BEADByUg+e1K5s/lEfRjb94l2b/xpqTlZgWYyrEyyE1+Bmhm2203A
+x/7/T95i1C3EclykR8k8nCpuQbWlb8O2zfiAVPkCGhGhd74rlHc4DzwHldYyHamK
+zxEJHAjrjMJVHyg6xPSxLxy7TrRJeMzy8VNgT6mROeA7V+WeFdSYaPoTTjAKPKuA
+N+H4+bPK2OmwHJ4V7/RdeiNdIjcnYLAOQmNrCamm5X0wHF9L+d3F/Ij+hL6CA4hM
+nhmzhZIRg9NGaQGyERKDpswcyjzHvfqYYsbqL0KrT2izstCx6yMWt3tMzCvsRx8j
+lZPN5fV3txtK2gc/RBA6fsDuMkWVRrLkht3bpOrnH+NMNYiG3V90QytvQtssH8El
+Py1S6v1hRPpQMsaK0wp1jNaksggE+HSBya1qSF3uofpqbUYEpHPUMXOJvNjUWt7E
+HEBomqVv5YeYNtSwSa/C+jYURhSthJ5oAJ6oB+cq4NFR8VgefIfKcUN+gAeVyrKc
+b5nHvhCeSR1sLA38k2+2Uyld79mhx+HE3Pd3TKZb8KJgqiccqpjXKbNMDox4CnSb
+Ll34ME/LTzRy2ZXbTnjIGI4jgSf6mNbIqIeLUrv+rf48ANTZWacrnbmjsjnN1Yxk
+nUzLlNMZfpyu+PqSHTaGvBY2Pa0/QtNDuWdozcGZlMhEZCD00kaetdqU3wARAQAB
+tDpYaWFveGlhbmcgWXUgKEtleSBmb3IgS3lsaW40LjAgcmVsZWFzZS4pIDx4eHl1
+QGFwYWNoZS5vcmc+iQI+BBMBAgAoBQJfVLB+AhsDBQkAdqcABgsJCAcDAgYVCAIJ
+CgsEFgIDAQIeAQIXgAAKCRCpp9bfr6c8HJ7GEACCoa8YscndLBfQHrZkmNWqMoNu
+LW9SYmy/Qyf3FgwU/Pual3cemntREASm9VFJg86gm9qigJPENFLtlJokOknOTXxg
+IuNXcRtVhIlckJBv8r5I++7vdXIiDoWWt4xcgd5irjx8/EAY3ztsFj33h8juEIWp
+qqA/8DunymqKGdamBySlwZoPdxrl26yuBF9wqxk7hOISaupfNn9AwJAq44A4t8M8
+hgnItvHEA7hG2iss2txIl/AEpIsjE1WHNubdDeQ07YaFDH5Tv20xgQMFU9aQkuy8
+EsrtbOlL6YG/UOKgAh5gtLP5+dIkDtjdyznzFql0LunwHF3HfjEcXWqTS+s12hJE
+/vZnn7MXrSLeZop9VkMMRWmQPRm2pn6Nimrm+5E5Y2/SulTVMqbjix6oiRnBcuvr
+W4IuAdo3egVKC+f9yvdUk2rWKU8kNj3mr8F2lvwp3htzhVD+SF7QtbhREKRPs1Ky
+TvvZUSdsnEog+r9C5pQ5Tk13uWE7Qj51DHrKiTL3fAgBxXCE29me4bPd0tB9VrvI
+aCwAnmUT2ij+Lr4msbgRvd7oXUdu/QMqs7alOADq8xLH9oFTaVswjGJHAgCvLgnG
+9ia3lbl0ADcPAeg4Zq/9mrCct5DjF5w8pl3gmXjt/AOEkmoUasoCmjexBpfBCR58
+wZbq602rYPX277/+K7kCDQRfVLB+ARAAyiiEMz/Fuyi1lYeF1nNm0a0D4V8hiaE8
+SIPa1Lnbcwcbpi5/upzGtEk9ug+ZwcqzfDx9fNumembaalSwKoZhLXW7NyRlcWKF
+KuUvFrJzJ9m7K3VqMan+AcMU51SodaJWijYp2N31x7eZT+NK8CualG3spBneHzek
+LPv2q7TcjrMczzZjdFtHrF7PSBDySPmtwfGe9tk1/s5A17iAgGac2GWZsGbahhYu
+8T1hRVwxJAiSEuIQp8oVn9q8698QLyZvy3KAySiLQTfmPSjYK12mKCcbcNkrhIev
+lFz1+x3DQzbKTSnLXSwoW7Yl8SHln6OY2z7JLGtWlZIram4h5Hk8p26JgX29qRLu
+IIC523Q+Nq02rSM+NyVjm7vUua9/6oYbyWved7SxmvJlwTFjr7sy9DWRgRWjPms7
+ObEECBLJKEW9gMNwnYL7GlQ5I9wU+1INdMPTNR/yyMwS1q0H6SnEPojhYczdBa9K
+4afLei6mCES+76KiSwKABtRe+JAJtHNJ2qx2MffdoeuaRTpQpQ6N0T6yMEkeyso5
+IeVLso6BsQJryQiZ09fxPjf/xL2Ip12UclFnTkydO3axU1DVlsUGglOdYJOcwZrQ
+/Tv7KFFj4eW6oKqrrgCOAYTA9zdE6jhWheJE8waXNZS2cZxdket4SZXwk/kwEpyd
+fRbwKdOLxu0AEQEAAYkCJQQYAQIADwUCX1SwfgIbDAUJAHanAAAKCRCpp9bfr6c8
+HNW3D/9M0anMSpDMkgxu8f3deocLMETF15eRr17QtTXGH5r26WQIoPdKsJdwO35y
+H04zp+ylNGMHbISzB6Zw1OfrP1iGUIWYEMFVR4l1wLMzwPUBkyT0q6hKZJMG06YJ
+p58ftShjZ53+fQ6KOP8JRIOkJoR/KA8viTEe5xSUcfS/GZZP7qNAjfiz96s22cNn
+6L/eG+QhOFA29pzpqUY2dEqpiwyDy6HF7EP+HXkAG9TnB8tIS9CYR22tEakJQMub
+p7Fx2DyydBErjJBSSDILMMW4NSMj7DsAH8WkadEcs2D/RIk49E+dCYyzycddxyOC
+PpeVcCdBVMTDw6FEVLKosytQDPqAAbmPlcQbwMcjRnO73CJnxrQoQFoDTFIxeDxE
+U82nFyDAbsHcouVDXM+9qNHqQBHXluTdmfqjLvUQyMGh+E575uKtMGB1fC7h3Vzm
+s+AEU+unsaPMMBoJgvyXcQjJIFfM/3VK09FlVBrutN9UPCQ4A/oJ7sNyVP7S0KTf
+GBcuw7Gzzz3BVG6Mmt4arGjjQRwiUsHZAwllIHFad/46xTJ8ttNikCDy4jUGm83K
+cSW7baYjWBs6p2Ia/8+EP05iCt3dYp8csrGkvURv9MwvfkLOIpEjMzkViTzndABI
+pDYU6GR1+p3hEgOwTYhd7oMrKb7zWQUX5gzcnKjKRqtHDQO1Gg==
+=wSHj
+-----END PGP PUBLIC KEY BLOCK-----
+pub 4096R/AFA73C1C 2020-09-06 [expires: 2020-12-05]
+uid Xiaoxiang Yu (Key for Kylin4.0 release.) <xx...@apache.org>
+sig 3 AFA73C1C 2020-09-06 Xiaoxiang Yu (Key for Kylin4.0 release.) <xx...@apache.org>
+sub 4096R/32EC2659 2020-09-06 [expires: 2020-12-05]
+sig AFA73C1C 2020-09-06 Xiaoxiang Yu (Key for Kylin4.0 release.) <xx...@apache.org>
\ No newline at end of file
diff --git a/build/conf/kylin-tools-log4j.properties b/build/conf/kylin-tools-log4j.properties
index 9c4e2bd..d73d92a 100644
--- a/build/conf/kylin-tools-log4j.properties
+++ b/build/conf/kylin-tools-log4j.properties
@@ -33,8 +33,9 @@ log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}:%L : %m%n
#log4j.logger.org.apache.hadoop=ERROR
-log4j.logger.org.apache.kylin=DEBUG
+log4j.logger.org.apache.kylin=INFO
log4j.logger.org.springframework=WARN
+log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.springframework.security=INFO
log4j.logger.org.apache.kylin.tool.shaded=INFO
log4j.logger.org.apache.spark=WARN
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 78ba123..72d5f83 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2875,6 +2875,10 @@ public abstract class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.query.pushdown.base-shuffle-partition-size", "48"));
}
+ public boolean isAutoStartSparder() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.auto-sparder-context", "true"));
+ }
+
// ============================================================================
// Spark with Kerberos
// ============================================================================
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 975d193..ddefabe 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -95,7 +95,7 @@ public class KylinVersion implements Comparable {
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
- private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("3.9.9.0001");
+ private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("4.0.0");
private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index 565d4e9..4eb2f17 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -17,7 +17,7 @@
### METADATA | ENV ###
-# The metadata store in hbase
+# The metadata store has two implementations(RDBMS/HBase), while RDBMS is recommended
kylin.metadata.url=kylin_metadata@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=XXXX,password=XXXXXX,maxActive=10,maxIdle=10
# metadata cache sync retry times
@@ -27,7 +27,7 @@ kylin.metadata.sync-retries=3
kylin.env.hdfs-working-dir=/kylin
# DEV|QA|PROD. DEV will turn on some dev features, QA and PROD has no difference in terms of functions.
-kylin.env=DEV
+kylin.env=QA
# kylin zk base path
kylin.env.zookeeper-base-path=/kylin
@@ -140,8 +140,6 @@ kylin.cube.algorithm.inmem-auto-optimize=true
kylin.cube.aggrgroup.max-combination=32768
-kylin.snapshot.max-mb=300
-
kylin.cube.cubeplanner.enabled=false
kylin.cube.cubeplanner.enabled-for-existing-cube=false
kylin.cube.cubeplanner.expansion-threshold=15.0
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 43c0d65..4212363 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>4.0.0-SNAPSHOT</version>
+ <version>4.0.0-beta-SNAPSHOT</version>
</parent>
<dependencies>
@@ -64,4 +64,16 @@
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
</project>
diff --git a/build/conf/kylin-tools-log4j.properties b/kylin-spark-project/build/conf/kylin-tools-log4j.properties
similarity index 73%
copy from build/conf/kylin-tools-log4j.properties
copy to kylin-spark-project/build/conf/kylin-tools-log4j.properties
index 9c4e2bd..7c43f1a 100644
--- a/build/conf/kylin-tools-log4j.properties
+++ b/kylin-spark-project/build/conf/kylin-tools-log4j.properties
@@ -17,13 +17,7 @@
#
-# the kylin-tools-log4j.properties is mainly for configuring log properties on kylin tools, including:
-# 1. tools launched by kylin.sh script, e.g. DeployCoprocessorCLI
-# 2. DebugTomcat
-# 3. others
-#
-# It's called kylin-tools-log4j.properties so that it won't distract users from the other more important log4j config file: kylin-server-log4j.properties
-# enable this by -Dlog4j.configuration=kylin-tools-log4j.properties
+## For surefire plugin to use
log4j.rootLogger=INFO,stderr
@@ -33,8 +27,9 @@ log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}:%L : %m%n
#log4j.logger.org.apache.hadoop=ERROR
-log4j.logger.org.apache.kylin=DEBUG
+log4j.logger.org.apache.kylin=INFO
log4j.logger.org.springframework=WARN
+log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.springframework.security=INFO
log4j.logger.org.apache.kylin.tool.shaded=INFO
log4j.logger.org.apache.spark=WARN
diff --git a/kylin-spark-project/kylin-spark-test/pom.xml b/kylin-spark-project/kylin-spark-test/pom.xml
index 398eb62..e775a9e 100644
--- a/kylin-spark-project/kylin-spark-test/pom.xml
+++ b/kylin-spark-project/kylin-spark-test/pom.xml
@@ -16,13 +16,11 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-spark-project</artifactId>
- <version>4.0.0-SNAPSHOT</version>
+ <version>4.0.0-beta-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -157,4 +155,16 @@
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
</project>
\ No newline at end of file
diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialSparderContext.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialSparderContext.java
index 05559e9..49714bd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialSparderContext.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialSparderContext.java
@@ -35,8 +35,9 @@ public class InitialSparderContext implements InitializingBean {
}
private void runInitialSparder() {
- if (ServerMode.isJobOnly(KylinConfig.getInstanceFromEnv())) {
- logger.info("This is job node, do not need to start Spark");
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ if (ServerMode.isJobOnly(config) || !config.isAutoStartSparder()) {
+ logger.info("Maybe this is job node, or switch is off, do not need to start Spark, {}", config.isAutoStartSparder());
return;
}
logger.info("Spark is starting.....");