You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/09/06 15:24:24 UTC
[kylin] branch kylin-on-parquet-v2 updated (a31a2b1 -> 524e2da)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git.
discard a31a2b1 KYLIN-4660 Fix QueryMetricsTest
new 524e2da KYLIN-4660 Fix QueryMetricsTest
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a31a2b1)
\
N -- N -- N refs/heads/kylin-on-parquet-v2 (524e2da)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
core-dictionary/pom.xml | 2 +-
kylin-spark-project/kylin-spark-test/pom.xml | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
[kylin] 01/01: KYLIN-4660 Fix QueryMetricsTest
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 524e2da496b86e22590cf8e46ab7766a819b1e25
Author: XiaoxiangYu <hi...@126.com>
AuthorDate: Sun Sep 6 23:24:07 2020 +0800
KYLIN-4660 Fix QueryMetricsTest
- QueryMetricsTest will not start a sparder context.
- Skip some module because I want to make sure UT pass at 30 mins.
---
.travis.yml | 4 +-
KEYS | 58 ++++++++++++++++++++++
build/conf/kylin-tools-log4j.properties | 3 +-
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++
.../java/org/apache/kylin/common/KylinVersion.java | 2 +-
.../src/main/resources/kylin-defaults.properties | 6 +--
core-dictionary/pom.xml | 12 +++++
.../build}/conf/kylin-tools-log4j.properties | 11 ++--
kylin-spark-project/kylin-spark-test/pom.xml | 12 +++++
.../kylin/rest/init/InitialSparderContext.java | 5 +-
10 files changed, 100 insertions(+), 17 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 6e53d12..cbb9598 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -39,7 +39,9 @@ before_script:
&& mvn -version
- echo "MAVEN_OPTS='-Xms1024m -Xmx3072m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m'" > ~/.mavenrc
- sed -i 's/log4j.logger.org.apache.kylin=DEBUG/log4j.logger.org.apache.kylin=WARN/g' build/conf/kylin-server-log4j.properties
- - sed -i 's/log4j.logger.org.apache.kylin=DEBUG/log4j.logger.org.apache.kylin=WARN/g' build/conf/kylin-tools-log4j.properties
+ - sed -i 's/log4j.logger.org.apache.kylin=INFO/log4j.logger.org.apache.kylin=WARN/g' build/conf/kylin-tools-log4j.properties
+ - sed -i 's/log4j.logger.org.apache.kylin=INFO/log4j.logger.org.apache.kylin=WARN/g' kylin-spark-project/build/conf/kylin-tools-log4j.properties
+
script:
- mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent test coveralls:report -e
diff --git a/KEYS b/KEYS
index 8c0fa0b..00ab8d0 100644
--- a/KEYS
+++ b/KEYS
@@ -478,3 +478,61 @@ mUdVyu44Yf7c3XnrZL0F7mwzFmpgeTWGOTzqfO5YSeHr5x8gizsBVADaoB1uPjPz
kzyYjjy3XzKvLlVIHuqksvDyibrlU1StUmNS64lfugMY1RfLERUpRdV3
=DvSV
-----END PGP PUBLIC KEY BLOCK-----
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2.0.14 (GNU/Linux)
+
+mQINBF9UsH4BEADByUg+e1K5s/lEfRjb94l2b/xpqTlZgWYyrEyyE1+Bmhm2203A
+x/7/T95i1C3EclykR8k8nCpuQbWlb8O2zfiAVPkCGhGhd74rlHc4DzwHldYyHamK
+zxEJHAjrjMJVHyg6xPSxLxy7TrRJeMzy8VNgT6mROeA7V+WeFdSYaPoTTjAKPKuA
+N+H4+bPK2OmwHJ4V7/RdeiNdIjcnYLAOQmNrCamm5X0wHF9L+d3F/Ij+hL6CA4hM
+nhmzhZIRg9NGaQGyERKDpswcyjzHvfqYYsbqL0KrT2izstCx6yMWt3tMzCvsRx8j
+lZPN5fV3txtK2gc/RBA6fsDuMkWVRrLkht3bpOrnH+NMNYiG3V90QytvQtssH8El
+Py1S6v1hRPpQMsaK0wp1jNaksggE+HSBya1qSF3uofpqbUYEpHPUMXOJvNjUWt7E
+HEBomqVv5YeYNtSwSa/C+jYURhSthJ5oAJ6oB+cq4NFR8VgefIfKcUN+gAeVyrKc
+b5nHvhCeSR1sLA38k2+2Uyld79mhx+HE3Pd3TKZb8KJgqiccqpjXKbNMDox4CnSb
+Ll34ME/LTzRy2ZXbTnjIGI4jgSf6mNbIqIeLUrv+rf48ANTZWacrnbmjsjnN1Yxk
+nUzLlNMZfpyu+PqSHTaGvBY2Pa0/QtNDuWdozcGZlMhEZCD00kaetdqU3wARAQAB
+tDpYaWFveGlhbmcgWXUgKEtleSBmb3IgS3lsaW40LjAgcmVsZWFzZS4pIDx4eHl1
+QGFwYWNoZS5vcmc+iQI+BBMBAgAoBQJfVLB+AhsDBQkAdqcABgsJCAcDAgYVCAIJ
+CgsEFgIDAQIeAQIXgAAKCRCpp9bfr6c8HJ7GEACCoa8YscndLBfQHrZkmNWqMoNu
+LW9SYmy/Qyf3FgwU/Pual3cemntREASm9VFJg86gm9qigJPENFLtlJokOknOTXxg
+IuNXcRtVhIlckJBv8r5I++7vdXIiDoWWt4xcgd5irjx8/EAY3ztsFj33h8juEIWp
+qqA/8DunymqKGdamBySlwZoPdxrl26yuBF9wqxk7hOISaupfNn9AwJAq44A4t8M8
+hgnItvHEA7hG2iss2txIl/AEpIsjE1WHNubdDeQ07YaFDH5Tv20xgQMFU9aQkuy8
+EsrtbOlL6YG/UOKgAh5gtLP5+dIkDtjdyznzFql0LunwHF3HfjEcXWqTS+s12hJE
+/vZnn7MXrSLeZop9VkMMRWmQPRm2pn6Nimrm+5E5Y2/SulTVMqbjix6oiRnBcuvr
+W4IuAdo3egVKC+f9yvdUk2rWKU8kNj3mr8F2lvwp3htzhVD+SF7QtbhREKRPs1Ky
+TvvZUSdsnEog+r9C5pQ5Tk13uWE7Qj51DHrKiTL3fAgBxXCE29me4bPd0tB9VrvI
+aCwAnmUT2ij+Lr4msbgRvd7oXUdu/QMqs7alOADq8xLH9oFTaVswjGJHAgCvLgnG
+9ia3lbl0ADcPAeg4Zq/9mrCct5DjF5w8pl3gmXjt/AOEkmoUasoCmjexBpfBCR58
+wZbq602rYPX277/+K7kCDQRfVLB+ARAAyiiEMz/Fuyi1lYeF1nNm0a0D4V8hiaE8
+SIPa1Lnbcwcbpi5/upzGtEk9ug+ZwcqzfDx9fNumembaalSwKoZhLXW7NyRlcWKF
+KuUvFrJzJ9m7K3VqMan+AcMU51SodaJWijYp2N31x7eZT+NK8CualG3spBneHzek
+LPv2q7TcjrMczzZjdFtHrF7PSBDySPmtwfGe9tk1/s5A17iAgGac2GWZsGbahhYu
+8T1hRVwxJAiSEuIQp8oVn9q8698QLyZvy3KAySiLQTfmPSjYK12mKCcbcNkrhIev
+lFz1+x3DQzbKTSnLXSwoW7Yl8SHln6OY2z7JLGtWlZIram4h5Hk8p26JgX29qRLu
+IIC523Q+Nq02rSM+NyVjm7vUua9/6oYbyWved7SxmvJlwTFjr7sy9DWRgRWjPms7
+ObEECBLJKEW9gMNwnYL7GlQ5I9wU+1INdMPTNR/yyMwS1q0H6SnEPojhYczdBa9K
+4afLei6mCES+76KiSwKABtRe+JAJtHNJ2qx2MffdoeuaRTpQpQ6N0T6yMEkeyso5
+IeVLso6BsQJryQiZ09fxPjf/xL2Ip12UclFnTkydO3axU1DVlsUGglOdYJOcwZrQ
+/Tv7KFFj4eW6oKqrrgCOAYTA9zdE6jhWheJE8waXNZS2cZxdket4SZXwk/kwEpyd
+fRbwKdOLxu0AEQEAAYkCJQQYAQIADwUCX1SwfgIbDAUJAHanAAAKCRCpp9bfr6c8
+HNW3D/9M0anMSpDMkgxu8f3deocLMETF15eRr17QtTXGH5r26WQIoPdKsJdwO35y
+H04zp+ylNGMHbISzB6Zw1OfrP1iGUIWYEMFVR4l1wLMzwPUBkyT0q6hKZJMG06YJ
+p58ftShjZ53+fQ6KOP8JRIOkJoR/KA8viTEe5xSUcfS/GZZP7qNAjfiz96s22cNn
+6L/eG+QhOFA29pzpqUY2dEqpiwyDy6HF7EP+HXkAG9TnB8tIS9CYR22tEakJQMub
+p7Fx2DyydBErjJBSSDILMMW4NSMj7DsAH8WkadEcs2D/RIk49E+dCYyzycddxyOC
+PpeVcCdBVMTDw6FEVLKosytQDPqAAbmPlcQbwMcjRnO73CJnxrQoQFoDTFIxeDxE
+U82nFyDAbsHcouVDXM+9qNHqQBHXluTdmfqjLvUQyMGh+E575uKtMGB1fC7h3Vzm
+s+AEU+unsaPMMBoJgvyXcQjJIFfM/3VK09FlVBrutN9UPCQ4A/oJ7sNyVP7S0KTf
+GBcuw7Gzzz3BVG6Mmt4arGjjQRwiUsHZAwllIHFad/46xTJ8ttNikCDy4jUGm83K
+cSW7baYjWBs6p2Ia/8+EP05iCt3dYp8csrGkvURv9MwvfkLOIpEjMzkViTzndABI
+pDYU6GR1+p3hEgOwTYhd7oMrKb7zWQUX5gzcnKjKRqtHDQO1Gg==
+=wSHj
+-----END PGP PUBLIC KEY BLOCK-----
+pub 4096R/AFA73C1C 2020-09-06 [expires: 2020-12-05]
+uid Xiaoxiang Yu (Key for Kylin4.0 release.) <xx...@apache.org>
+sig 3 AFA73C1C 2020-09-06 Xiaoxiang Yu (Key for Kylin4.0 release.) <xx...@apache.org>
+sub 4096R/32EC2659 2020-09-06 [expires: 2020-12-05]
+sig AFA73C1C 2020-09-06 Xiaoxiang Yu (Key for Kylin4.0 release.) <xx...@apache.org>
\ No newline at end of file
diff --git a/build/conf/kylin-tools-log4j.properties b/build/conf/kylin-tools-log4j.properties
index 9c4e2bd..d73d92a 100644
--- a/build/conf/kylin-tools-log4j.properties
+++ b/build/conf/kylin-tools-log4j.properties
@@ -33,8 +33,9 @@ log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}:%L : %m%n
#log4j.logger.org.apache.hadoop=ERROR
-log4j.logger.org.apache.kylin=DEBUG
+log4j.logger.org.apache.kylin=INFO
log4j.logger.org.springframework=WARN
+log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.springframework.security=INFO
log4j.logger.org.apache.kylin.tool.shaded=INFO
log4j.logger.org.apache.spark=WARN
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 78ba123..72d5f83 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2875,6 +2875,10 @@ public abstract class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.query.pushdown.base-shuffle-partition-size", "48"));
}
+ public boolean isAutoStartSparder() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.auto-sparder-context", "true"));
+ }
+
// ============================================================================
// Spark with Kerberos
// ============================================================================
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 975d193..ddefabe 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -95,7 +95,7 @@ public class KylinVersion implements Comparable {
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
- private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("3.9.9.0001");
+ private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("4.0.0");
private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index 565d4e9..4eb2f17 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -17,7 +17,7 @@
### METADATA | ENV ###
-# The metadata store in hbase
+# The metadata store has two implementations(RDBMS/HBase), while RDBMS is recommended
kylin.metadata.url=kylin_metadata@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=XXXX,password=XXXXXX,maxActive=10,maxIdle=10
# metadata cache sync retry times
@@ -27,7 +27,7 @@ kylin.metadata.sync-retries=3
kylin.env.hdfs-working-dir=/kylin
# DEV|QA|PROD. DEV will turn on some dev features, QA and PROD has no difference in terms of functions.
-kylin.env=DEV
+kylin.env=QA
# kylin zk base path
kylin.env.zookeeper-base-path=/kylin
@@ -140,8 +140,6 @@ kylin.cube.algorithm.inmem-auto-optimize=true
kylin.cube.aggrgroup.max-combination=32768
-kylin.snapshot.max-mb=300
-
kylin.cube.cubeplanner.enabled=false
kylin.cube.cubeplanner.enabled-for-existing-cube=false
kylin.cube.cubeplanner.expansion-threshold=15.0
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 43c0d65..6c5ee1c 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -64,4 +64,16 @@
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
</project>
diff --git a/build/conf/kylin-tools-log4j.properties b/kylin-spark-project/build/conf/kylin-tools-log4j.properties
similarity index 73%
copy from build/conf/kylin-tools-log4j.properties
copy to kylin-spark-project/build/conf/kylin-tools-log4j.properties
index 9c4e2bd..7c43f1a 100644
--- a/build/conf/kylin-tools-log4j.properties
+++ b/kylin-spark-project/build/conf/kylin-tools-log4j.properties
@@ -17,13 +17,7 @@
#
-# the kylin-tools-log4j.properties is mainly for configuring log properties on kylin tools, including:
-# 1. tools launched by kylin.sh script, e.g. DeployCoprocessorCLI
-# 2. DebugTomcat
-# 3. others
-#
-# It's called kylin-tools-log4j.properties so that it won't distract users from the other more important log4j config file: kylin-server-log4j.properties
-# enable this by -Dlog4j.configuration=kylin-tools-log4j.properties
+## For surefire plugin to use
log4j.rootLogger=INFO,stderr
@@ -33,8 +27,9 @@ log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}:%L : %m%n
#log4j.logger.org.apache.hadoop=ERROR
-log4j.logger.org.apache.kylin=DEBUG
+log4j.logger.org.apache.kylin=INFO
log4j.logger.org.springframework=WARN
+log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.springframework.security=INFO
log4j.logger.org.apache.kylin.tool.shaded=INFO
log4j.logger.org.apache.spark=WARN
diff --git a/kylin-spark-project/kylin-spark-test/pom.xml b/kylin-spark-project/kylin-spark-test/pom.xml
index 398eb62..5541dd9 100644
--- a/kylin-spark-project/kylin-spark-test/pom.xml
+++ b/kylin-spark-project/kylin-spark-test/pom.xml
@@ -157,4 +157,16 @@
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
</project>
\ No newline at end of file
diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialSparderContext.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialSparderContext.java
index 05559e9..49714bd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialSparderContext.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialSparderContext.java
@@ -35,8 +35,9 @@ public class InitialSparderContext implements InitializingBean {
}
private void runInitialSparder() {
- if (ServerMode.isJobOnly(KylinConfig.getInstanceFromEnv())) {
- logger.info("This is job node, do not need to start Spark");
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ if (ServerMode.isJobOnly(config) || !config.isAutoStartSparder()) {
+ logger.info("Maybe this is job node, or switch is off, do not need to start Spark, {}", config.isAutoStartSparder());
return;
}
logger.info("Spark is starting.....");