You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2024/03/29 10:32:08 UTC
(kylin) branch kylin5 updated (8118afb449 -> bcd4220ef2)
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a change to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
from 8118afb449 KYLIN-5683 tree list load more columns issue
new 3ec3063135 KYLIN-5732 Upgrade snappy-java to fix vulnerabilities
new 387e68d0a6 KYLIN-5733 TDS template modify to English
new 6816947de4 KYLIN-5762 Initialize job scheduler encounters NPE
new b15367e3e2 KYLIN-5760 Support matching model exactly for list jobs
new d008c28156 KYLIN-5737 Fix vulnerabilities, upgrade tomcat-embed-core & netty (#30573)
new c396134127 KYLIN-5742 Make the query result of duplicate group sets same as Spark
new 1d665a8282 KYLIN-5762 Initialize job scheduler encounters NPE
new 0e3354414d KYLIN-5740 Equivalent aggregations matches index failed
new 7cd3fd0ecc KYLIN-5741 change all epoch when empty projects request
new 5532198301 KYLIN-5743 optimization of SumExpression prevents ProjectJoinTransposeRule to hit AggIndex
new 572d78a45b KYLIN-5756 Concat string and number with + gives unexpected result
new d13199db74 KYLIN-5744 Answering snapshot first, then answering by metadata
new 227e7a0683 KYLIN-5741 punctuation optimized
new 3cbcead2ed KYLIN-5746 model list get optimize fix
new bcd4220ef2 update spark version to 3.2.0-kylin-4.6.12.0
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../kap/secondstorage/metadata/TableFlow.java | 13 ++
pom.xml | 39 +++---
.../kylin/rest/controller/NEpochController.java | 9 +-
.../rest/controller/NEpochControllerTest.java | 19 +--
.../apache/kylin/rest/KylinPrepareEnvListener.java | 14 ++-
.../apache/kylin/rest/service/ProjectService.java | 8 ++
.../kylin/rest/service/AccessServiceTest.java | 4 +-
.../kylin/rest/service/AclTCRServiceTest.java | 2 +-
.../org/apache/kylin/common/KylinConfigBase.java | 14 ++-
.../apache/kylin/common/constant/Constants.java | 1 +
.../org/apache/kylin/common/msg/CnMessage.java | 6 +
.../java/org/apache/kylin/common/msg/Message.java | 7 ++
.../kylin/job/execution/ExecutableContext.java | 9 +-
.../kylin/job/execution/ExecutableThread.java | 1 +
.../job/impl/threadpool/NDefaultScheduler.java | 21 ++--
.../apache/kylin/job/runners/FetcherRunner.java | 3 +-
.../org/apache/kylin/job/runners/JobRunner.java | 3 +-
.../kylin/job/execution/ExecutableContextTest.java | 4 +-
.../job/execution/NExecutableManagerTest.java | 2 +-
.../job/impl/threadpool/NDefaultSchedulerTest.java | 21 ++++
.../metadata/cube/cuboid/AggIndexMatcher.java | 13 +-
.../metadata/project/NProjectManagerTest.java | 2 +-
.../kylin/metrics/HdfsCapacityMetricsTest.java | 2 +-
.../kylin/rest/controller/JobController.java | 4 +-
.../kylin/rest/controller/v2/JobControllerV2.java | 2 +-
.../kylin/rest/controller/JobControllerTest.java | 6 +-
.../rest/controller/v2/JobControllerV2Test.java | 11 +-
.../org/apache/kylin/rest/request/JobFilter.java | 2 +
.../org/apache/kylin/rest/service/JobService.java | 5 +
.../kylin/rest/service/DagJobServiceTest.java | 8 +-
.../apache/kylin/rest/service/JobServiceTest.java | 49 ++++++--
.../localmeta/data/DEFAULT.TEST_DIM_AS_MEASURE.csv | 3 +
.../metadata/_global/project/dim_as_measure.json} | 2 +-
.../f29d1a92-c115-ec68-5575-9cfcc0d65890.json | 63 ++++++++++
.../4409fb82-8f3a-ba01-9852-0b05f1fe690f.json | 64 ++++++++++
.../f29d1a92-c115-ec68-5575-9cfcc0d65890.json} | 59 ++++-----
.../f29d1a92-c115-ec68-5575-9cfcc0d65890.json} | 61 ++++-----
.../table/DEFAULT.TEST_DIM_AS_MEASURE.json} | 40 +++---
.../newten/table/DEFAULT.TEST_DIM_AS_MEASURE.json} | 40 +++---
src/kylin-it/pom.xml | 9 --
.../kylin/query/engine/QueryExecWithMetaTest.java | 2 +-
.../kylin/query/engine/SumExprPlannerTest.java | 11 ++
.../query/routing/QueryLayoutChooserTest.java | 47 ++++++-
.../kylin/query/engine/SumExprPlannerTest.xml | 32 +++++
src/kylin-server-it/pom.xml | 9 --
.../kylin/rest/service/MetaStoreService.java | 24 ++--
.../apache/kylin/rest/service/ModelService.java | 17 +--
.../bisync/tds/tableau.connector.template.xml | 2 +-
.../main/resources/bisync/tds/tableau.template.xml | 2 +-
.../kylin/rest/service/ProjectServiceTest.java | 32 ++++-
.../bisync_tableau/nmodel_basic_all_cols.tds | 2 +-
.../bisync_tableau/nmodel_basic_inner_all_cols.tds | 2 +-
.../nmodel_full_measure_test.connector.tds | 2 +-
.../nmodel_full_measure_test.connector_admin.tds | 2 +-
.../nmodel_full_measure_test.connector_cc.tds | 2 +-
...nmodel_full_measure_test.connector_cc_admin.tds | 2 +-
...del_full_measure_test.connector_hierarchies.tds | 2 +-
..._full_measure_test.connector_no_hierarchies.tds | 2 +-
...odel_full_measure_test.connector_permission.tds | 2 +-
...ure_test.connector_permission_agg_index_col.tds | 2 +-
...l_measure_test.connector_permission_all_col.tds | 2 +-
...easure_test.connector_permission_no_measure.tds | 2 +-
.../nmodel_full_measure_test.connector_user.tds | 2 +-
...l_measure_test.connector_user_agg_index_col.tds | 2 +-
...del_full_measure_test.table_index_connector.tds | 2 +-
.../bisync_tableau/nmodel_full_measure_test.tds | 2 +-
src/query-common/pom.xml | 4 +
.../kylin/query/relnode/KapAggregateRel.java | 58 ++++-----
.../apache/kylin/query/relnode/OLAPContext.java | 24 ++--
.../rest/service/QueryHistoryServiceTest.java | 2 +-
.../apache/kylin/query/engine/PlannerFactory.java | 14 ++-
.../kylin/query/engine/AsyncQueryJobTest.java | 2 +-
src/server/pom.xml | 1 +
.../org/apache/spark/sql/udf/CeilFloorTest.scala | 139 +++++++++++----------
.../org/apache/spark/sql/SparderTypeUtil.scala | 2 +-
src/streaming/pom.xml | 7 +-
.../apache/kylin/helper/MetadataToolHelper.java | 20 +--
77 files changed, 740 insertions(+), 388 deletions(-)
create mode 100644 src/examples/test_case_data/localmeta/data/DEFAULT.TEST_DIM_AS_MEASURE.csv
copy src/{kylin-it/src/test/resources/ut_meta/sum_lc/metadata/_global.project/sum_lc.json => examples/test_case_data/localmeta/metadata/_global/project/dim_as_measure.json} (64%)
create mode 100644 src/examples/test_case_data/localmeta/metadata/dim_as_measure/dataflow/f29d1a92-c115-ec68-5575-9cfcc0d65890.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/dim_as_measure/dataflow_details/f29d1a92-c115-ec68-5575-9cfcc0d65890/4409fb82-8f3a-ba01-9852-0b05f1fe690f.json
copy src/examples/test_case_data/localmeta/metadata/{special_character_in_column/index_plan/8c08822f-296a-b097-c910-e38d8934b6f9.json => dim_as_measure/index_plan/f29d1a92-c115-ec68-5575-9cfcc0d65890.json} (50%)
copy src/{kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/648098d6-3009-5b26-3e20-82e494cfdb0c.json => examples/test_case_data/localmeta/metadata/dim_as_measure/model_desc/f29d1a92-c115-ec68-5575-9cfcc0d65890.json} (57%)
copy src/{kylin-it/src/test/resources/ut_meta/enhanced_agg_pushdown/metadata/test_agg_pushdown/table/DEFAULT.TEST_ORDER.json => examples/test_case_data/localmeta/metadata/dim_as_measure/table/DEFAULT.TEST_DIM_AS_MEASURE.json} (50%)
copy src/{kylin-it/src/test/resources/ut_meta/enhanced_agg_pushdown/metadata/test_agg_pushdown/table/DEFAULT.TEST_ORDER.json => examples/test_case_data/localmeta/metadata/newten/table/DEFAULT.TEST_DIM_AS_MEASURE.json} (50%)
(kylin) 15/15: update spark version to 3.2.0-kylin-4.6.12.0
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit bcd4220ef2556eaab318d7e2fa4d6e3c1d9fe312
Author: Yu Gan <yu...@kyligence.io>
AuthorDate: Sat Jul 8 10:08:47 2023 +0800
update spark version to 3.2.0-kylin-4.6.12.0
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 8bfc1dd630..9b68f4a83d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@
<!-- Spark versions -->
<delta.version>2.0.2</delta.version>
- <spark.version>3.2.0-kylin-4.6.9.0</spark.version>
+ <spark.version>3.2.0-kylin-4.6.12.0</spark.version>
<roaring.version>0.9.2-kylin-r4</roaring.version>
(kylin) 01/15: KYLIN-5732 Upgrade snappy-java to fix vulnerabilities
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3ec3063135f6b99cb4f7934d056a6ca7fd4a1e5e
Author: Yinghao Lin <39...@users.noreply.github.com>
AuthorDate: Tue Jun 20 16:23:34 2023 +0800
KYLIN-5732 Upgrade snappy-java to fix vulnerabilities
---
pom.xml | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/pom.xml b/pom.xml
index 534e8bcca6..904f18cb80 100644
--- a/pom.xml
+++ b/pom.xml
@@ -371,6 +371,7 @@
<jackson-databind.version>2.13.4.2</jackson-databind.version>
<nimbus-jose-jwt.version>9.23</nimbus-jose-jwt.version>
<system-lambda.version>1.2.1</system-lambda.version>
+ <snappy-java.version>1.1.10.1</snappy-java.version>
</properties>
<modules>
@@ -2954,6 +2955,13 @@
<version>${system-lambda.version}</version>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>${snappy-java.version}</version>
+ </dependency>
+
</dependencies>
</dependencyManagement>
(kylin) 10/15: KYLIN-5743 optimization of SumExpression prevents ProjectJoinTransposeRule to hit AggIndex
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 553219830193ec43f7b6d68fe626b04456633120
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Thu Jun 29 14:39:05 2023 +0800
KYLIN-5743 optimization of SumExpression prevents ProjectJoinTransposeRule to hit AggIndex
---
.../kylin/query/engine/SumExprPlannerTest.java | 11 ++++++++
.../kylin/query/engine/SumExprPlannerTest.xml | 32 ++++++++++++++++++++++
.../apache/kylin/query/engine/PlannerFactory.java | 14 ++++++----
3 files changed, 51 insertions(+), 6 deletions(-)
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
index 7e00b6cfb2..9692ec4f4e 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
@@ -199,6 +199,17 @@ public class SumExprPlannerTest extends CalciteRuleTestBase {
KapJoinRule.INSTANCE, KapProjectRule.INSTANCE));
}
+ @Test
+ public void testSumExpressionSkipProjectJoinTransposeRule() {
+ String sql = "select sum( case when LO_COMMITDATE = '20230501' then LO_DISCOUNT end )\n" //
+ + "from (\n" //
+ + " select LO_COMMITDATE, LO_DISCOUNT, P_LINEORDER.LO_ORDERDATE from ssb.P_LINEORDER\n" //
+ + " ) a\n" //
+ + "where LO_COMMITDATE = ( select max(LO_COMMITDATE) from ssb.P_LINEORDER )";
+ openSumCaseWhen();
+ checkSQL("ssb", sql, null, null);
+ }
+
@Test
public void testAggPushdownWithCrossJoin1() {
diff --git a/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml b/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
index e7b560f9a5..0796314ef7 100644
--- a/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
+++ b/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
@@ -1377,6 +1377,38 @@ KapOLAPToEnumerableConverter
]]>
</Resource>
</TestCase>
+ <TestCase name="testSumExpressionSkipProjectJoinTransposeRule">
+ <Resource name="planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[]], groups=[null], EXPR$0=[SUM($0)], ctx=[])
+ KapProjectRel($f0=[CASE(=($0, 20230501), $1, null)], ctx=[])
+ KapJoinRel(condition=[=($0, $3)], joinType=[inner], ctx=[])
+ KapProjectRel(LO_COMMITDATE=[$15], LO_DISCOUNT=[$11], LO_ORDERDATE=[$5], ctx=[])
+ KapTableScan(table=[[SSB, P_LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]])
+ KapAggregateRel(group-set=[[]], groups=[null], EXPR$0=[MAX($0)], ctx=[])
+ KapProjectRel(LO_COMMITDATE=[$15], ctx=[])
+ KapTableScan(table=[[SSB, P_LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[]], groups=[null], AGG$0=[SUM($0)], ctx=[])
+ KapProjectRel($f0=[CASE(=($0, 20230501), $1, null)], ctx=[])
+ KapAggregateRel(group-set=[[0]], groups=[null], TOP_AGG$0=[SUM($1)], TOP_AGG$1=[SUM($2)], ctx=[])
+ KapProjectRel(LO_COMMITDATE=[$0], SUM_CASE$0$0=[$1], $f2=[*(0, $2)], ctx=[])
+ KapAggregateRel(group-set=[[0]], groups=[null], SUM_CASE$0$0=[SUM($1)], SUM_CONST$1=[COUNT()], ctx=[])
+ KapProjectRel(LO_COMMITDATE=[$0], LO_DISCOUNT=[$1], ctx=[])
+ KapJoinRel(condition=[=($0, $3)], joinType=[inner], ctx=[])
+ KapProjectRel(LO_COMMITDATE=[$15], LO_DISCOUNT=[$11], LO_ORDERDATE=[$5], ctx=[])
+ KapTableScan(table=[[SSB, P_LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]])
+ KapAggregateRel(group-set=[[]], groups=[null], EXPR$0=[MAX($0)], ctx=[])
+ KapProjectRel(LO_COMMITDATE=[$15], ctx=[])
+ KapTableScan(table=[[SSB, P_LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]])
+]]>
+ </Resource>
+ </TestCase>
<TestCase name="testAggPushdownWithCrossJoin2">
<Resource name="planBefore">
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java b/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
index 8616d6e14a..bac1fbf81b 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
@@ -65,10 +65,13 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.guava30.shaded.common.base.Function;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.query.engine.meta.PlannerContext;
+import org.apache.kylin.query.relnode.OLAPContext;
+
import io.kyligence.kap.query.optrule.AggregateMultipleExpandRule;
import io.kyligence.kap.query.optrule.AggregateProjectReduceRule;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.engine.meta.PlannerContext;
import io.kyligence.kap.query.optrule.CorrReduceFunctionRule;
import io.kyligence.kap.query.optrule.KAPValuesRule;
import io.kyligence.kap.query.optrule.KapAggregateReduceFunctionsRule;
@@ -89,9 +92,6 @@ import io.kyligence.kap.query.optrule.KapWindowRule;
import io.kyligence.kap.query.optrule.RightJoinToLeftJoinRule;
import io.kyligence.kap.query.optrule.SumConstantConvertRule;
-import org.apache.kylin.guava30.shaded.common.base.Function;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
-
/**
* factory that create optimizers and register opt rules
* TODO sort and register only necessary rules
@@ -240,7 +240,9 @@ public class PlannerFactory {
// UnionMergeRule may slow volcano planner optimization on large number of union clause
planner.removeRule(UnionMergeRule.INSTANCE);
- planner.addRule(KapProjectJoinTransposeRule.INSTANCE);
+ if (!kylinConfig.isConvertSumExpressionEnabled()) {
+ planner.addRule(KapProjectJoinTransposeRule.INSTANCE);
+ }
planner.removeRule(ProjectRemoveRule.INSTANCE);
// skip corr expandsion during model suggestion
(kylin) 05/15: KYLIN-5737 Fix vulnerabilities, upgrade tomcat-embed-core & netty (#30573)
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d008c28156b0b2cfea530b08f765613575b2c60e
Author: Yinghao Lin <39...@users.noreply.github.com>
AuthorDate: Thu Jun 29 20:09:22 2023 +0800
KYLIN-5737 Fix vulnerabilities, upgrade tomcat-embed-core & netty (#30573)
---
pom.xml | 27 ++++++++++-----------------
src/kylin-it/pom.xml | 9 ---------
src/kylin-server-it/pom.xml | 9 ---------
src/server/pom.xml | 1 +
src/streaming/pom.xml | 7 +------
5 files changed, 12 insertions(+), 41 deletions(-)
diff --git a/pom.xml b/pom.xml
index 904f18cb80..a61ca736d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,7 +100,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<arthas.version>3.6.7-kyarthas-r3</arthas.version>
- <tomcat-embed.version>9.0.74</tomcat-embed.version>
+ <tomcat-embed.version>9.0.76</tomcat-embed.version>
<!-- Keep same order with dependencyManagement -->
@@ -360,9 +360,8 @@
<derby.version>10.14.2.0</derby.version>
<jcommander.version>1.75</jcommander.version>
<ant.version>1.10.11</ant.version>
- <netty-all.version>4.1.89.Final</netty-all.version>
- <netty.version>3.10.6.Final</netty.version>
- <netty-codec.version>4.1.89.Final</netty-codec.version>
+ <netty3.version>3.10.6.Final</netty3.version>
+ <netty-handler.version>4.1.94.Final</netty-handler.version>
<servo-core.version>0.13.2</servo-core.version>
<apache-api.version>1.0.3</apache-api.version>
<saas.nacos.discovery.starter.version>2.6.2</saas.nacos.discovery.starter.version>
@@ -741,11 +740,6 @@
<artifactId>kylin-soft-affinity-cache</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec-haproxy</artifactId>
- <version>${netty-codec.version}</version>
- </dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
@@ -2788,14 +2782,8 @@
<!--spark 2.4-->
<dependency>
<groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- <version>${netty-all.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>${netty.version}</version>
- <scope>provided</scope>
+ <artifactId>netty-handler</artifactId>
+ <version>${netty-handler.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
@@ -2961,6 +2949,11 @@
<artifactId>snappy-java</artifactId>
<version>${snappy-java.version}</version>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>${netty3.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/src/kylin-it/pom.xml b/src/kylin-it/pom.xml
index 01282cf457..4272d86bf6 100644
--- a/src/kylin-it/pom.xml
+++ b/src/kylin-it/pom.xml
@@ -97,15 +97,6 @@
<artifactId>commons-cli</artifactId>
</dependency>
<!--for spark2.4-->
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>
diff --git a/src/kylin-server-it/pom.xml b/src/kylin-server-it/pom.xml
index 9c60c7bb48..8cd256e983 100644
--- a/src/kylin-server-it/pom.xml
+++ b/src/kylin-server-it/pom.xml
@@ -106,15 +106,6 @@
<artifactId>commons-cli</artifactId>
</dependency>
<!--for spark2.4-->
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>
diff --git a/src/server/pom.xml b/src/server/pom.xml
index fdf7d8182e..deb02b3ade 100644
--- a/src/server/pom.xml
+++ b/src/server/pom.xml
@@ -443,6 +443,7 @@
net.sf.py4j,
com.h2database,
jdk.tools,
+ io.netty
</excludeGroupIds>
<excludeArtifactIds>
unused
diff --git a/src/streaming/pom.xml b/src/streaming/pom.xml
index 105755ad83..e03dc67f0c 100644
--- a/src/streaming/pom.xml
+++ b/src/streaming/pom.xml
@@ -159,11 +159,6 @@
<scope>test</scope>
<version>3.4.0</version>
</dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -206,4 +201,4 @@
</plugin>
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
(kylin) 12/15: KYLIN-5744 Answering snapshot first, then answering by metadata
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d13199db7407552226a58060ceb9b41a7f859c36
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Thu Jun 29 15:28:35 2023 +0800
KYLIN-5744 Answering snapshot first, then answering by metadata
Hitting snapshot has a higher priority than
answering min/max with metadata.
Change the default behavior of routing query to metadata to true
1. Fix timezone problem.
Given the fact of min(cal_dt) = '2012-01-01',
the result of `select min(cal_dt) from test_kylin_fact`
may give '2011-12-31' rather than '2012-01-01'.
2. Fix min/max query hit a index without any dimensions.
For example: `select min(cal_dt) as min_cal_dt from test_kylin_fact `
hits index just has a measure of min(cal_dt),
the SparkPlan is wrong: ` project --- project --- tableScan `;
but the desired SparkPlan is: ` aggregate --- project --- tableScan `.
---
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../kylin/query/engine/QueryExecWithMetaTest.java | 2 +-
src/query-common/pom.xml | 4 ++
.../kylin/query/relnode/KapAggregateRel.java | 58 +++++++++-------------
.../apache/kylin/query/relnode/OLAPContext.java | 24 +++++----
.../org/apache/spark/sql/SparderTypeUtil.scala | 2 +-
6 files changed, 46 insertions(+), 46 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 03c1c06e89..cbd82ed108 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1734,7 +1734,7 @@ public abstract class KylinConfigBase implements Serializable {
// ============================================================================
public boolean isRouteToMetadataEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.using-metadata-answer-minmax-of-dimension", FALSE));
+ return Boolean.parseBoolean(this.getOptional("kylin.query.using-metadata-answer-minmax-of-dimension", TRUE));
}
public boolean partialMatchNonEquiJoins() {
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
index ac84de6f06..94d98eb5ea 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
@@ -230,7 +230,7 @@ public class QueryExecWithMetaTest extends NLocalWithSparkSessionTest {
List<String> result = iterator.next();
Assert.assertEquals("2147483648,21474836483289,2132,2147483647,-128,127,0,9,0.0,10000.0,"
+ "0.3255242,85208.3241,10.0000,201.3235,abc,xyz,aaaaaaaa,xxxxxxxxxxxxxxxxxxxxx,abcd,zzzz,"
- + "2000-12-31,2004-04-16,2004-04-01 00:00:00,2004-04-17 00:32:23.032,false,true,"
+ + "2001-01-01,2004-04-17,2004-04-01 00:00:00,2004-04-17 00:32:23.032,false,true,"
+ "null,null,null,null,null,null,null,null,null,null,null,null", String.join(",", result));
}
diff --git a/src/query-common/pom.xml b/src/query-common/pom.xml
index d6af343c34..16af9a8028 100644
--- a/src/query-common/pom.xml
+++ b/src/query-common/pom.xml
@@ -55,6 +55,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kap-second-storage-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-spark-common</artifactId>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
index f44272beb5..5a064dd840 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
@@ -45,6 +45,9 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.measure.corr.CorrMeasureType;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.model.NDataflow;
@@ -55,10 +58,6 @@ import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.util.ICutContextStrategy;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
/**
*
*/
@@ -69,7 +68,7 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
private ImmutableList<Integer> rewriteGroupKeys; // preserve the ordering of group keys after CC replacement
private List<ImmutableBitSet> rewriteGroupSets; // group sets with cc replaced
List<AggregateCall> aggregateCalls;
- private Set<TblColRef> groupByInnerColumns = new HashSet<>(); // inner columns in group keys, for CC generation
+ private final Set<TblColRef> groupByInnerColumns = new HashSet<>(); // inner columns in group keys, for CC generation
private Set<OLAPContext> subContexts = Sets.newHashSet();
public KapAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, boolean indicator,
@@ -203,8 +202,8 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
TblColRef originalColumn = inputColumnRowType.getColumnByIndex(i);
if (null != this.context && this.context.getGroupCCColRewriteMapping().containsKey(originalColumn)) {
groups.add(this.context.getGroupCCColRewriteMapping().get(originalColumn));
- groupKeys.add(inputColumnRowType
- .getIndexByName(this.context.getGroupCCColRewriteMapping().get(originalColumn).getName()));
+ String colName = this.context.getGroupCCColRewriteMapping().get(originalColumn).getName();
+ groupKeys.add(inputColumnRowType.getIndexByName(colName));
} else {
Set<TblColRef> sourceColumns = inputColumnRowType.getSourceColumnsByIndex(i);
groups.addAll(sourceColumns);
@@ -283,17 +282,13 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
this.rewriteAggCalls = new ArrayList<>(aggregateCalls.size());
for (int i = 0; i < this.aggregateCalls.size(); i++) {
AggregateCall aggCall = this.aggregateCalls.get(i);
- if (SqlStdOperatorTable.GROUPING == aggCall.getAggregation()) {
+ if (SqlStdOperatorTable.GROUPING == aggCall.getAggregation()
+ || this.aggregations.get(i).isAggregateOnConstant()) {
this.rewriteAggCalls.add(aggCall);
continue;
}
- FunctionDesc cubeFunc = this.aggregations.get(i);
- if (cubeFunc.isAggregateOnConstant()) {
- this.rewriteAggCalls.add(aggCall);
- continue;
- }
- aggCall = rewriteAggCall(aggCall, cubeFunc);
+ aggCall = rewriteAggCall(aggCall, this.aggregations.get(i));
this.rewriteAggCalls.add(aggCall);
}
getContext().setExactlyAggregate(isExactlyMatched());
@@ -329,8 +324,9 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
return false;
}
- if (!checkAggCall())
+ if (!checkAggCall()) {
return false;
+ }
Set<String> cuboidDimSet = new HashSet<>();
if (getContext() != null && getContext().storageContext.getCandidate() != null) {
cuboidDimSet = getContext().storageContext.getCandidate().getLayoutEntity().getOrderedDimensions().values()
@@ -343,6 +339,9 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
OLAPRel.logger.info("cuboid dimensions: {}", cuboidDimSet);
boolean isDimensionMatch = isDimExactlyMatch(groupByCols, cuboidDimSet);
+ if (KylinConfig.getInstanceFromEnv().isRouteToMetadataEnabled()) {
+ isDimensionMatch = isDimensionMatch && !groupByCols.isEmpty();
+ }
if (!isDimensionMatch) {
return false;
}
@@ -359,17 +358,15 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
&& dataflow.getQueryableSegments().get(0).getMultiPartitions().size() <= 1;
}
- private Boolean checkAggCall() {
+ private boolean checkAggCall() {
for (AggregateCall call : getRewriteAggCalls()) {
if (!supportedFunction.contains(OLAPAggregateRel.getAggrFuncName(call))) {
return false;
}
// bitmap uuid is fine with exactly matched cube as what we need to query
// from the cube is exactly the binary bitmap
- if (OLAPAggregateRel.getAggrFuncName(call).equals("BITMAP_UUID")) {
- continue;
- }
- if (OLAPAggregateRel.getAggrFuncName(call).equals(FunctionDesc.FUNC_BITMAP_BUILD)) {
+ if (OLAPAggregateRel.getAggrFuncName(call).equals("BITMAP_UUID")
+ || OLAPAggregateRel.getAggrFuncName(call).equals(FunctionDesc.FUNC_BITMAP_BUILD)) {
continue;
}
if (call instanceof KylinAggregateCall) {
@@ -417,11 +414,6 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
this.subContexts = contexts;
}
- @Override
- protected void buildRewriteFieldsAndMetricsColumns() {
- super.buildRewriteFieldsAndMetricsColumns();
- }
-
/**
* optimize its Context Rel after context cut off according some rules
* 1. push through the Agg Above Join Rel
@@ -430,7 +422,7 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
// case 1: Agg push through Join
if (context == null) {
for (OLAPContext subContext : subContexts) {
- if (subContext.aggregations.size() > 0)
+ if (!subContext.aggregations.isEmpty())
continue;
if (ContextUtil.qualifiedForAggInfoPushDown(this, subContext)) {
subContext.setTopNode(this);
@@ -444,16 +436,14 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
if (this.context == null)
return;
- for (TblColRef colRef : this.context.getGroupByColumns()) {
- if (!colRef.getName().startsWith("_KY_") && context.belongToContextTables(colRef))
- this.context.allColumns.add(colRef);
- }
+ this.context.getGroupByColumns().stream()
+ .filter(colRef -> !colRef.getName().startsWith("_KY_") && context.belongToContextTables(colRef))
+ .forEach(colRef -> this.context.allColumns.add(colRef));
if (!(getInput() instanceof KapProjectRel)) {
- for (TblColRef colRef : ((KapRel) getInput()).getColumnRowType().getAllColumns()) {
- if (context.belongToContextTables(colRef) && !colRef.getName().startsWith("_KY_"))
- context.allColumns.add(colRef);
- }
+ ((KapRel) getInput()).getColumnRowType().getAllColumns().stream()
+ .filter(colRef -> context.belongToContextTables(colRef) && !colRef.getName().startsWith("_KY_"))
+ .forEach(colRef -> context.allColumns.add(colRef));
return;
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index eadb698c06..5835ab21b1 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -34,6 +34,7 @@ import org.apache.calcite.rel.AbstractRelNode;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
@@ -47,6 +48,7 @@ import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.model.DimensionRangeInfo;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -60,11 +62,12 @@ import org.apache.kylin.metadata.query.QueryMetrics;
import org.apache.kylin.metadata.realization.HybridRealization;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.query.routing.RealizationCheck;
import org.apache.kylin.query.schema.OLAPSchema;
+import org.apache.kylin.query.schema.OLAPTable;
import org.apache.kylin.storage.StorageContext;
+import org.apache.spark.sql.util.SparderTypeUtil;
import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -590,9 +593,6 @@ public class OLAPContext {
public String genExecFunc(OLAPRel rel, String tableName) {
setReturnTupleInfo(rel.getRowType(), rel.getColumnRowType());
- if (canMinMaxDimAnsweredByMetadata(rel)) {
- return "executeMetadataQuery";
- }
if (isConstantQueryWithAggregations()) {
return "executeSimpleAggregationQuery";
@@ -603,6 +603,10 @@ public class OLAPContext {
return "executeLookupTableQuery";
}
+ if (canMinMaxDimAnsweredByMetadata(rel)) {
+ return "executeMetadataQuery";
+ }
+
return "executeOLAPQuery";
}
@@ -681,6 +685,7 @@ public class OLAPContext {
List<TblColRef> colRefs = tableScan.getColumnRowType().getAllColumns();
allFields.addAll(colRefs);
});
+ RelDataTypeFactory typeFactory = this.getTopNode().getCluster().getTypeFactory();
List<Object[]> result = new ArrayList<>();
for (NDataSegment segment : ((NDataflow) realization).getSegments()) {
if (segment.getStatus() != SegmentStatusEnum.READY) {
@@ -690,16 +695,17 @@ public class OLAPContext {
Object[] minList = new Object[allFields.size()];
Object[] maxList = new Object[allFields.size()];
for (TblColRef col : cols) {
- String dataType = col.getColumnDesc().getUpgradedType().getName();
int colId = allFields.indexOf(col);
String tblColRefIndex = getTblColRefIndex(col, realization);
- DimensionRangeInfo dimensionRangeInfo = infoMap.get(tblColRefIndex);
- if (dimensionRangeInfo == null) {
+ DimensionRangeInfo rangeInfo = infoMap.get(tblColRefIndex);
+ if (rangeInfo == null) {
minList[colId] = null;
maxList[colId] = null;
} else {
- minList[colId] = Tuple.convertOptiqCellValue(dimensionRangeInfo.getMin(), dataType);
- maxList[colId] = Tuple.convertOptiqCellValue(dimensionRangeInfo.getMax(), dataType);
+ ColumnDesc c = col.getColumnDesc();
+ RelDataType sqlType = OLAPTable.createSqlType(typeFactory, c.getUpgradedType(), c.isNullable());
+ minList[colId] = SparderTypeUtil.convertToStringWithCalciteType(rangeInfo.getMin(), sqlType, false);
+ maxList[colId] = SparderTypeUtil.convertToStringWithCalciteType(rangeInfo.getMax(), sqlType, false);
}
}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
index 79d4aaf451..eeda408931 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
@@ -129,7 +129,7 @@ object SparderTypeUtil extends Logging {
case "decimal" => new java.math.BigDecimal(s.toString)
case "date" => new java.sql.Date(DateFormat.stringToMillis(s.toString))
case "time" | "timestamp" | "datetime" =>
- val l = java.lang.Long.parseLong(s.toString)
+ val l = DateFormat.stringToMillis(s.toString)
Timestamp.valueOf(DateFormat.castTimestampToString(l))
case "tinyint" => s.toString.toByte
case "smallint" => s.toString.toShort
(kylin) 11/15: KYLIN-5756 Concat string and number with + gives unexpected result
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 572d78a45b6e0cabb248b6772a9b434221d9f323
Author: Zhiting Guo <35...@users.noreply.github.com>
AuthorDate: Thu Jun 29 18:00:09 2023 +0800
KYLIN-5756 Concat string and number with + gives unexpected result
KYLIN-5747 duplicated
Co-authored-by: Zhiting Guo <zh...@kyligence.io>
---
pom.xml | 2 +-
.../org/apache/kylin/rest/KylinPrepareEnvListener.java | 14 ++++++++++----
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 12 ++++++++++++
3 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/pom.xml b/pom.xml
index 83adcd3e33..8bfc1dd630 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,7 +135,7 @@
<scala-retry>0.3.0</scala-retry>
<!-- Calcite Version -->
- <calcite.version>1.116.0-kylin-4.x-r031</calcite.version>
+ <calcite.version>1.116.0-kylin-4.x-r033</calcite.version>
<avatica.version>4.x_1.10-r01</avatica.version>
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
index e700fb0801..a49c67e33b 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
@@ -24,6 +24,7 @@ import java.sql.SQLException;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.TempMetadataBuilder;
import org.apache.kylin.common.util.TimeZoneUtils;
@@ -75,14 +76,19 @@ public class KylinPrepareEnvListener implements EnvironmentPostProcessor, Ordere
Unsafe.setProperty("needCheckCC", "true");
val config = KylinConfig.getInstanceFromEnv();
if (config.isCalciteInClauseEnabled()) {
- Unsafe.setProperty("calcite.keep-in-clause", "true");
+ Unsafe.setProperty("calcite.keep-in-clause", KylinConfigBase.TRUE);
} else {
- Unsafe.setProperty("calcite.keep-in-clause", "false");
+ Unsafe.setProperty("calcite.keep-in-clause", KylinConfigBase.FALSE);
}
if (config.isCalciteConvertMultipleColumnsIntoOrEnabled()) {
- Unsafe.setProperty("calcite.convert-multiple-columns-in-to-or", "true");
+ Unsafe.setProperty("calcite.convert-multiple-columns-in-to-or", KylinConfigBase.TRUE);
} else {
- Unsafe.setProperty("calcite.convert-multiple-columns-in-to-or", "false");
+ Unsafe.setProperty("calcite.convert-multiple-columns-in-to-or", KylinConfigBase.FALSE);
+ }
+ if (config.isCalciteCompatibleWithMsSqlPlusEnabled()) {
+ Unsafe.setProperty("calcite.compatible-with-mssql-plus", KylinConfigBase.TRUE);
+ } else {
+ Unsafe.setProperty("calcite.compatible-with-mssql-plus", KylinConfigBase.FALSE);
}
Unsafe.setProperty("calcite.bindable.cache.maxSize", Integer.toString(config.getCalciteBindableCacheSize()));
Unsafe.setProperty("calcite.bindable.cache.concurrencyLevel",
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2908f05571..03c1c06e89 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2533,6 +2533,10 @@ public abstract class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.query.calcite.bindable.cache.concurrencyLevel", "5"));
}
+ public boolean isCalciteCompatibleWithMsSqlPlusEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.calcite.compatible-with-mssql-plus-enabled", FALSE));
+ }
+
public int getEventPollIntervalSecond() {
return Integer.parseInt(getOptional("kylin.job.event.poll-interval-second", "60"));
}
@@ -2823,6 +2827,14 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.smart.conf.skip-corr-reduce-rule", FALSE));
}
+ public int getOptimizeTransformerMaxIterations() {
+ return Integer.parseInt(getOptional("kylin.smart.conf.optimize-transformer-max-iterations", "2"));
+ }
+
+ public int getOptimizeTransformerConditionCountThreshold() {
+ return Integer.parseInt(getOptional("kylin.smart.conf.optimize-transformer-condition-count-threshold", "200"));
+ }
+
public String getEngineWriteFs() {
String engineWriteFs = getOptional("kylin.env.engine-write-fs", "");
return StringHelper.dropSuffix(engineWriteFs, File.separator);
(kylin) 14/15: KYLIN-5746 model list get optimize fix
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3cbcead2ed5cb3046647ef2a6617c9480254b1b9
Author: liang.huang <83...@users.noreply.github.com>
AuthorDate: Tue Jul 4 17:58:53 2023 +0800
KYLIN-5746 model list get optimize fix
---
.../apache/kylin/rest/service/ModelService.java | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 1ad9ce6cd1..59cc0a9203 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -372,10 +372,10 @@ public class ModelService extends AbstractModelService implements TableModelSupp
private void addOldSegmentParams(NDataModel model, NDataModelOldParams oldParams,
List<AbstractExecutable> executables) {
- if (((model instanceof NDataModelResponse) && !(model instanceof NDataModelLiteResponse)) || model.isFusionModel()) {
- List<NDataSegmentResponse> segments = getSegmentsResponse(model.getId(), model.getProject(), "1",
- String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true);
- calculateRecordSizeAndCount(segments, oldParams);
+ List<NDataSegmentResponse> segments = getSegmentsResponse(model.getId(), model.getProject(), "1",
+ String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true, (model instanceof NDataModelLiteResponse));
+ calculateRecordSizeAndCount(segments, oldParams);
+ if (model instanceof NDataModelResponse) {
((NDataModelResponse) model).setSegments(segments);
((NDataModelResponse) model).setHasSegments(
((NDataModelResponse) model).isHasSegments() || CollectionUtils.isNotEmpty(segments));
@@ -387,7 +387,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
NDataModel batchModel = fusionModel.getBatchModel();
if (!batchModel.isBroken()) {
List<NDataSegmentResponse> batchSegments = getSegmentsResponse(batchModel.getUuid(), model.getProject(),
- "1", String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true);
+ "1", String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true, false);
calculateRecordSizeAndCount(batchSegments, oldParams);
if (model instanceof FusionModelResponse) {
((FusionModelResponse) model).setBatchSegments(batchSegments);
@@ -925,9 +925,9 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
public List<NDataSegmentResponse> getSegmentsResponse(String modelId, String project, String start, String end,
- String status, List<AbstractExecutable> executables, String sortBy, boolean reverse) {
+ String status, List<AbstractExecutable> executables, String sortBy, boolean reverse, boolean lite) {
return getSegmentsResponse(modelId, project, start, end, status, null, null, executables, false, sortBy,
- reverse, null, null);
+ reverse, lite, null, null);
}
private List<AbstractExecutable> getAllRunningExecutable(String project) {
@@ -949,12 +949,12 @@ public class ModelService extends AbstractModelService implements TableModelSupp
String sortBy, boolean reverse, List<String> statuses, List<String> secondStorageStatuses) {
val executables = getPartialRunningExecutable(project, modelId);
return getSegmentsResponse(modelId, project, start, end, status, withAllIndexes, withoutAnyIndexes, executables,
- allToComplement, sortBy, reverse, statuses, secondStorageStatuses);
+ allToComplement, sortBy, reverse, false, statuses, secondStorageStatuses);
}
public List<NDataSegmentResponse> getSegmentsResponse(String modelId, String project, String start, String end,
String status, Collection<Long> withAllIndexes, Collection<Long> withoutAnyIndexes,
- List<AbstractExecutable> executables, boolean allToComplement, String sortBy, boolean reverse,
+ List<AbstractExecutable> executables, boolean allToComplement, String sortBy, boolean reverse, boolean lite,
List<String> statuses, List<String> secondStorageStatuses) {
aclEvaluate.checkProjectReadPermission(project);
NDataflowManager dataflowManager = getManager(NDataflowManager.class, project);
@@ -962,7 +962,9 @@ public class ModelService extends AbstractModelService implements TableModelSupp
List<NDataSegmentResponse> segmentResponseList = getSegmentsResponseCore(modelId, project, start, end, status,
withAllIndexes, withoutAnyIndexes, executables, allToComplement, dataflow);
addSecondStorageResponse(modelId, project, segmentResponseList, dataflow);
- addSecondStorageDisplayStatus(modelId, project, segmentResponseList);
+ if (!lite) {
+ addSecondStorageDisplayStatus(modelId, project, segmentResponseList);
+ }
changeSegmentDisplayStatus(modelId, project, segmentResponseList);
segmentResponseList = segmentResponseFilter(statuses, secondStorageStatuses, segmentResponseList, modelId,
project);
(kylin) 02/15: KYLIN-5733 TDS template modify to English
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 387e68d0a669b84d269341a8a4127c5b70d6bf55
Author: fanfanAlice <41...@users.noreply.github.com>
AuthorDate: Tue Jun 27 17:20:58 2023 +0800
KYLIN-5733 TDS template modify to English
---
.../src/main/resources/bisync/tds/tableau.connector.template.xml | 2 +-
src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml | 2 +-
.../src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds | 2 +-
.../src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds | 2 +-
.../resources/bisync_tableau/nmodel_full_measure_test.connector.tds | 2 +-
.../bisync_tableau/nmodel_full_measure_test.connector_admin.tds | 2 +-
.../resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds | 2 +-
.../bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds | 2 +-
.../bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds | 2 +-
.../nmodel_full_measure_test.connector_no_hierarchies.tds | 2 +-
.../bisync_tableau/nmodel_full_measure_test.connector_permission.tds | 2 +-
.../nmodel_full_measure_test.connector_permission_agg_index_col.tds | 2 +-
.../nmodel_full_measure_test.connector_permission_all_col.tds | 2 +-
.../nmodel_full_measure_test.connector_permission_no_measure.tds | 2 +-
.../bisync_tableau/nmodel_full_measure_test.connector_user.tds | 2 +-
.../nmodel_full_measure_test.connector_user_agg_index_col.tds | 2 +-
.../bisync_tableau/nmodel_full_measure_test.table_index_connector.tds | 2 +-
.../src/test/resources/bisync_tableau/nmodel_full_measure_test.tds | 2 +-
18 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml b/src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml
index f410cfdb71..a83940f9b6 100755
--- a/src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml
+++ b/src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml
@@ -25,6 +25,6 @@
</connection>
<aliases enabled="yes"/>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml b/src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml
index a72b454dfc..16a2ef2b7f 100644
--- a/src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml
+++ b/src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml
@@ -27,6 +27,6 @@
</connection>
<aliases enabled='yes' />
<semantic-values>
- <semantic-value key='[Country].[Name]' value='"美国"' />
+ <semantic-value key='[Country].[Name]' value='"US"' />
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds
index f1a91e41f4..6477036203 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds
@@ -544,6 +544,6 @@
</column>
<drill-paths/>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds
index 6b84dddcbb..edf95da259 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds
@@ -541,6 +541,6 @@
</drill-path>
</drill-paths>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds
index ac3e77ff62..7b0a31f9b9 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds
@@ -120,6 +120,6 @@
</column>
<drill-paths/>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_admin.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_admin.tds
index f02667d2a2..ceaad43004 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_admin.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_admin.tds
@@ -120,6 +120,6 @@
</column>
<drill-paths/>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds
index 4663e549bc..91017c63d1 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds
@@ -159,6 +159,6 @@
</drill-path>
</drill-paths>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds
index 258f0d9bbc..c53824e688 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds
@@ -541,6 +541,6 @@
</drill-path>
</drill-paths>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds
index 2557069476..7a4112f188 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds
@@ -138,6 +138,6 @@
</drill-path>
</drill-paths>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_no_hierarchies.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_no_hierarchies.tds
index 4630c4e508..691af9c246 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_no_hierarchies.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_no_hierarchies.tds
@@ -132,6 +132,6 @@
</column>
<drill-paths/>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission.tds
index 2557069476..7a4112f188 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission.tds
@@ -138,6 +138,6 @@
</drill-path>
</drill-paths>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds
index 0b6bda410f..86b7c1468c 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds
@@ -514,6 +514,6 @@
</drill-path>
</drill-paths>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_all_col.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_all_col.tds
index 645eea4536..6a830b6d2b 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_all_col.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_all_col.tds
@@ -514,6 +514,6 @@
</drill-path>
</drill-paths>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_no_measure.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_no_measure.tds
index 7b7cc684f3..a6657a5818 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_no_measure.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_no_measure.tds
@@ -514,6 +514,6 @@
</drill-path>
</drill-paths>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_user.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_user.tds
index 3509aaabec..c7d321a767 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_user.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_user.tds
@@ -48,6 +48,6 @@
</column>
<drill-paths/>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_user_agg_index_col.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_user_agg_index_col.tds
index 5b0c0c8073..52b7fe7544 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_user_agg_index_col.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_user_agg_index_col.tds
@@ -48,6 +48,6 @@
</column>
<drill-paths/>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.table_index_connector.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.table_index_connector.tds
index 8b36f698be..9804c59461 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.table_index_connector.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.table_index_connector.tds
@@ -544,6 +544,6 @@
</column>
<drill-paths/>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.tds
index 2975098b96..bdea4f1ad7 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.tds
@@ -120,6 +120,6 @@
</column>
<drill-paths/>
<semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
+ <semantic-value key="[Country].[Name]" value=""US""/>
</semantic-values>
</datasource>
(kylin) 03/15: KYLIN-5762 Initialize job scheduler encounters NPE
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6816947de43cc1f3a7101c06a1d32d623a36ef06
Author: sibingzhang <74...@users.noreply.github.com>
AuthorDate: Wed Jun 28 18:28:54 2023 +0800
KYLIN-5762 Initialize job scheduler encounters NPE
Co-authored-by: sibing.zhang <si...@qq.com>
---
.../java/org/apache/kylin/job/execution/ExecutableContext.java | 9 ++++++---
.../java/org/apache/kylin/job/execution/ExecutableThread.java | 1 +
.../src/main/java/org/apache/kylin/job/runners/JobRunner.java | 3 ++-
.../org/apache/kylin/job/execution/ExecutableContextTest.java | 4 +++-
.../org/apache/kylin/job/execution/NExecutableManagerTest.java | 2 +-
.../java/org/apache/kylin/rest/service/DagJobServiceTest.java | 8 ++++----
6 files changed, 17 insertions(+), 10 deletions(-)
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
index 16c9e9f576..fea0a74f09 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
@@ -23,10 +23,10 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.common.KylinConfig;
-
import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@@ -68,7 +68,6 @@ public class ExecutableContext {
// Only used when the job starts scheduling
public void addRunningJob(Executable executable) {
- runningJobThreads.put(executable.getId(), Thread.currentThread());
runningJobs.put(executable.getId(), executable);
runningJobInfos.put(executable.getId(), System.currentTimeMillis());
}
@@ -84,6 +83,10 @@ public class ExecutableContext {
return runningJobThreads.get(executable.getId());
}
+ public void addRunningJobThread(Executable executable) {
+ runningJobThreads.put(executable.getId(), Thread.currentThread());
+ }
+
public Map<String, Executable> getRunningJobs() {
return Collections.unmodifiableMap(runningJobs);
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableThread.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableThread.java
index 61140daeec..162eed1daa 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableThread.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableThread.java
@@ -50,6 +50,7 @@ public class ExecutableThread extends Thread {
try (SetThreadName ignored = new SetThreadName("JobWorker(project:%s,jobid:%s)", project, jobIdSimple);
SetLogCategory logCategory = new SetLogCategory("schedule")) {
context.addRunningJob(executable);
+ context.addRunningJobThread(executable);
dagExecutable.executeDagExecutable(dagExecutablesMap, executable, context);
} finally {
context.removeRunningJob(executable);
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/runners/JobRunner.java b/src/core-job/src/main/java/org/apache/kylin/job/runners/JobRunner.java
index 3e839f57f4..b8cd19bed0 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/runners/JobRunner.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/runners/JobRunner.java
@@ -18,13 +18,13 @@
package org.apache.kylin.job.runners;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.logging.SetLogCategory;
import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.exception.JobStoppedNonVoluntarilyException;
import org.apache.kylin.job.exception.JobStoppedVoluntarilyException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
-import org.apache.kylin.common.logging.SetLogCategory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,6 +50,7 @@ public class JobRunner extends AbstractDefaultSchedulerRunner {
val jobIdSimple = executable.getId().substring(0, 8);
try (SetThreadName ignored = new SetThreadName("JobWorker(project:%s,jobid:%s)", project, jobIdSimple);
SetLogCategory logCategory = new SetLogCategory("schedule")) {
+ context.addRunningJobThread(executable);
executable.execute(context);
// trigger the next step asap
fetcherRunner.scheduleNext();
diff --git a/src/core-job/src/test/java/org/apache/kylin/job/execution/ExecutableContextTest.java b/src/core-job/src/test/java/org/apache/kylin/job/execution/ExecutableContextTest.java
index db4e2ff2b0..b294d0458f 100644
--- a/src/core-job/src/test/java/org/apache/kylin/job/execution/ExecutableContextTest.java
+++ b/src/core-job/src/test/java/org/apache/kylin/job/execution/ExecutableContextTest.java
@@ -24,10 +24,10 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.junit.jupiter.api.Test;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
import lombok.val;
@MetadataInfo
@@ -39,7 +39,9 @@ class ExecutableContextTest {
val context = new ExecutableContext(Maps.newConcurrentMap(), Maps.newConcurrentMap(),
KylinConfig.getInstanceFromEnv(), 0);
context.addRunningJob(job);
+ assertNull(context.getRunningJobThread(job));
+ context.addRunningJobThread(job);
assertNotNull(context.getRunningJobThread(job));
context.removeRunningJob(job);
diff --git a/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java b/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java
index d7a891c74d..e56c8f260e 100644
--- a/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java
+++ b/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java
@@ -861,7 +861,7 @@ public class NExecutableManagerTest extends NLocalFileMetadataTestCase {
new Thread(() -> {
try {
- scheduler.getContext().addRunningJob(job);
+ scheduler.getContext().addRunningJobThread(job);
job.doWork(scheduler.getContext());
} catch (ExecuteException ignored) {
} finally {
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/DagJobServiceTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/DagJobServiceTest.java
index 30e9b0380a..3417887425 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/DagJobServiceTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/DagJobServiceTest.java
@@ -27,6 +27,8 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.engine.spark.job.NSparkExecutable;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.job.constant.JobActionEnum;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -54,8 +56,6 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.springframework.test.util.ReflectionTestUtils;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import lombok.val;
@MetadataInfo
@@ -275,7 +275,7 @@ class DagJobServiceTest {
}
@Test
- void updateStepStatus() throws Exception {
+ void updateStepStatus() {
val config = KylinConfig.getInstanceFromEnv();
val sparkMaster = config.getSparkMaster();
val scheduler = NDefaultScheduler.getInstance(DEFAULT_PROJECT);
@@ -290,7 +290,7 @@ class DagJobServiceTest {
executable.killApplicationIfExistsOrUpdateStepStatus();
Assertions.assertNull(context.getRunningJobThread(executable));
- context.addRunningJob(executable);
+ context.addRunningJobThread(executable);
Assertions.assertNotNull(context.getRunningJobThread(executable));
executable.killApplicationIfExistsOrUpdateStepStatus();
Assertions.assertNull(context.getRunningJobThread(executable));
(kylin) 07/15: KYLIN-5762 Initialize job scheduler encounters NPE
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1d665a82829a82fc9a51181378aa87de1a98dea2
Author: huangsheng <hu...@163.com>
AuthorDate: Fri Jun 30 10:43:50 2023 +0800
KYLIN-5762 Initialize job scheduler encounters NPE
---
.../apache/kylin/rest/service/ProjectService.java | 8 +++++++
.../apache/kylin/common/constant/Constants.java | 1 +
.../org/apache/kylin/common/msg/CnMessage.java | 6 +++++
.../java/org/apache/kylin/common/msg/Message.java | 7 ++++++
.../job/impl/threadpool/NDefaultScheduler.java | 21 ++++++++++-------
.../apache/kylin/job/runners/FetcherRunner.java | 3 +--
.../job/impl/threadpool/NDefaultSchedulerTest.java | 21 +++++++++++++++++
.../kylin/rest/service/ProjectServiceTest.java | 26 ++++++++++++++++++++++
8 files changed, 83 insertions(+), 10 deletions(-)
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index d32d8b261e..416f577e0e 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -25,6 +25,7 @@ import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_PASS_
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_ENABLE_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_NAME_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_USER_KEY;
+import static org.apache.kylin.common.constant.Constants.KYLIN_JOB_MAX_CONCURRENT_JOBS;
import static org.apache.kylin.common.constant.NonCustomProjectLevelConfig.DATASOURCE_TYPE;
import static org.apache.kylin.common.exception.ServerErrorCode.DATABASE_NOT_EXIST;
import static org.apache.kylin.common.exception.ServerErrorCode.DUPLICATE_PROJECT_NAME;
@@ -82,6 +83,7 @@ import org.apache.kylin.guava30.shaded.common.base.Strings;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.guava30.shaded.common.primitives.Ints;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.NExecutableManager;
@@ -431,6 +433,12 @@ public class ProjectService extends BasicService {
if (projectInstance == null) {
throw new KylinException(PROJECT_NOT_EXIST, project);
}
+ if (overrideKylinProps.containsKey(KYLIN_JOB_MAX_CONCURRENT_JOBS)) {
+ val maxConcurrentJobs = Ints.tryParse(overrideKylinProps.get(KYLIN_JOB_MAX_CONCURRENT_JOBS));
+ if (Objects.isNull(maxConcurrentJobs) || maxConcurrentJobs < 0)
+ throw new KylinException(INVALID_PARAMETER,
+ MsgPicker.getMsg().getIllegalNegative(KYLIN_JOB_MAX_CONCURRENT_JOBS));
+ }
encryptJdbcPassInOverrideKylinProps(overrideKylinProps);
projectManager.updateProject(project, copyForWrite -> copyForWrite.getOverrideKylinProps()
.putAll(KylinConfig.trimKVFromMap(overrideKylinProps)));
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java b/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
index a49d012fa2..6f3a9e1f13 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
@@ -48,6 +48,7 @@ public class Constants {
public static final String KYLIN_SOURCE_JDBC_CONNECTION_URL_KEY = "kylin.source.jdbc.connection-url";
public static final String KYLIN_SOURCE_JDBC_USER_KEY = "kylin.source.jdbc.user";
public static final String KYLIN_SOURCE_JDBC_DRIVER_KEY = "kylin.source.jdbc.driver";
+ public static final String KYLIN_JOB_MAX_CONCURRENT_JOBS = "kylin.job.max-concurrent-jobs";
public static final String UNLIMITED = "Unlimited";
public static final String TRACE_ID = "traceId";
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index c62f52156d..ed626ec5d1 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -32,6 +32,7 @@ public class CnMessage extends Message {
private static final String TASK_TIMEOUT = "执行超时";
private static final String PARAMETER_EMPTY = "请输入参数 “%s” 的值。";
+ private static final String PARAMETER_MUST_BE_POSITIVE_NUMBER= "参数 %s 的值必须为非负数.";
protected CnMessage() {
@@ -1719,4 +1720,9 @@ public class CnMessage extends Message {
public String getQueryNotRunningError() {
return "该查询没有在运行，请检查";
}
+
+ @Override
+ public String getIllegalNegative(String parameter) {
+ return String.format(PARAMETER_MUST_BE_POSITIVE_NUMBER, parameter);
+ }
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index 79e267eab1..3dff08d642 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -47,6 +47,8 @@ public class Message {
private static final String PROFILING_COLLECT_TIMEOUT = "Async profiler timeout";
private static final String PARAMETER_EMPTY = "Please enter the value for the parameter '%s'.";
+ private static final String PARAMETER_MUST_BE_POSITIVE_NUMBER = "The value of %s must be a positive number.";
+
private static final String DDL_UNSUPPORTED = "Unsupported DDL syntax, only support single `create view`, `drop view`, `alter view`, `show create table`";
private static final String DDL_VIEW_NAME_ERROR = "View names need to start with KYLIN_";
private static final String DDL_VIEW_NAME_DUPLICATE_ERROR = "Logical View names is duplicate";
@@ -1621,4 +1623,9 @@ public class Message {
return "Query is not running, please check.";
}
+ public String getIllegalNegative(String parameter) {
+ return String.format(PARAMETER_MUST_BE_POSITIVE_NUMBER, parameter);
+
+ }
+
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/NDefaultScheduler.java b/src/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/NDefaultScheduler.java
index 368785b51a..6d3d2e5742 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/NDefaultScheduler.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/NDefaultScheduler.java
@@ -29,13 +29,16 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import lombok.Setter;
import org.apache.commons.lang3.RandomUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.ExecutorServiceUtil;
import org.apache.kylin.common.util.NamedThreadFactory;
import org.apache.kylin.common.util.SystemInfoCollector;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.base.Strings;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.job.Scheduler;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -43,18 +46,14 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.NExecutableManager;
import org.apache.kylin.job.runners.FetcherRunner;
import org.apache.kylin.job.runners.JobCheckRunner;
+import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.base.Strings;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
-import org.apache.kylin.metadata.epoch.EpochManager;
import lombok.Getter;
+import lombok.Setter;
import lombok.SneakyThrows;
import lombok.val;
@@ -153,7 +152,7 @@ public class NDefaultScheduler implements Scheduler<AbstractExecutable> {
//load all executable, set them to a consistent status
fetcherPool = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("FetchJobWorker(project:" + project + ")"));
- int corePoolSize = getMaxConcurrentJobLimitByProject(config, jobEngineConfig, project);
+ int corePoolSize = getMaxConcurrentJobLimitByProjectForInitThread(config, jobEngineConfig, project);
if (config.getAutoSetConcurrentJob()) {
val availableMemoryRate = config.getMaxLocalConsumptionRatio();
synchronized (NDefaultScheduler.class) {
@@ -232,6 +231,12 @@ public class NDefaultScheduler implements Scheduler<AbstractExecutable> {
return 1.0 * memoryRemaining.availablePermits();
}
+ public int getMaxConcurrentJobLimitByProjectForInitThread(KylinConfig config, JobEngineConfig jobEngineConfig,
+ String project) {
+ int jobLimit = getMaxConcurrentJobLimitByProject(config, jobEngineConfig, project);
+ return jobLimit <= 0 ? 1 : jobLimit;
+ }
+
public int getMaxConcurrentJobLimitByProject(KylinConfig config, JobEngineConfig jobEngineConfig, String project) {
ProjectInstance prjInstance = NProjectManager.getInstance(config).getProject(project);
if (Strings.isNullOrEmpty(project) || prjInstance == null) {
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/runners/FetcherRunner.java b/src/core-job/src/main/java/org/apache/kylin/job/runners/FetcherRunner.java
index 4494992d8c..31434a4761 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/runners/FetcherRunner.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/runners/FetcherRunner.java
@@ -80,7 +80,6 @@ public class FetcherRunner extends AbstractDefaultSchedulerRunner {
}
return false;
}
-
private boolean markErrorJob(String jobId) {
try {
return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
@@ -252,7 +251,7 @@ public class FetcherRunner extends AbstractDefaultSchedulerRunner {
private void checkAndUpdateJobPoolNum() {
final ThreadPoolExecutor pool = (ThreadPoolExecutor) jobPool;
int maximumPoolSize = pool.getMaximumPoolSize();
- int maxConcurrentJobLimit = nDefaultScheduler.getMaxConcurrentJobLimitByProject(context.getConfig(),
+ int maxConcurrentJobLimit = nDefaultScheduler.getMaxConcurrentJobLimitByProjectForInitThread(context.getConfig(),
nDefaultScheduler.getJobEngineConfig(), project);
int activeCount = pool.getActiveCount();
if (maximumPoolSize == maxConcurrentJobLimit) {
diff --git a/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/NDefaultSchedulerTest.java b/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/NDefaultSchedulerTest.java
index 3ab266d35e..f47060e229 100644
--- a/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/NDefaultSchedulerTest.java
+++ b/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/NDefaultSchedulerTest.java
@@ -1445,6 +1445,27 @@ public class NDefaultSchedulerTest extends BaseSchedulerTest {
scheduler.shutdown();
Assert.assertEquals(memory, NDefaultScheduler.getMemoryRemaining().availablePermits());
+
+ config.setProperty("kylin.job.max-concurrent-jobs", "0");
+ scheduler.init(new JobEngineConfig(config));
+ val memory2 = NDefaultScheduler.getMemoryRemaining().availablePermits();
+ val df2 = NDataflowManager.getInstance(getTestConfig(), project).getDataflow(modelId);
+ val job3 = generateJob(df2, project);
+ val job4 = generatePartial(df2, project);
+ executableManager.addJob(job3);
+ executableManager.addJob(job4);
+ val runningExecutables2 = executableManager.getRunningExecutables(project, modelId);
+ runningExecutables.sort(Comparator.comparing(AbstractExecutable::getCreateTime));
+ Assert.assertEquals(ExecutableState.READY, runningExecutables2.get(0).getStatus());
+ Assert.assertEquals(ExecutableState.READY, runningExecutables2.get(1).getStatus());
+ config.setProperty("kylin.job.max-concurrent-jobs", "2");
+ waitForJobByStatus(job3.getId(), 60000, null, executableManager);
+ waitForJobByStatus(job4.getId(), 60000, null, executableManager);
+ Assert.assertEquals(ExecutableState.SUCCEED, executableManager.getOutput(job1.getId()).getState());
+ Assert.assertEquals(ExecutableState.SUCCEED, executableManager.getOutput(job2.getId()).getState());
+
+ scheduler.shutdown();
+ Assert.assertEquals(memory, NDefaultScheduler.getMemoryRemaining().availablePermits());
}
@Test
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
index 029b3c4fda..f47a64020b 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
@@ -529,6 +529,7 @@ public class ProjectServiceTest extends NLocalFileMetadataTestCase {
testOverrideP.put(" testk1 ", " testv1 ");
testOverrideP.put("tes tk2", "test v2");
testOverrideP.put(" tes tk3 ", " t estv3 ");
+ testOverrideP.put("kylin.job.max-concurrent-jobs", "10");
projectService.updateProjectConfig(PROJECT, testOverrideP);
@@ -537,6 +538,31 @@ public class ProjectServiceTest extends NLocalFileMetadataTestCase {
Assert.assertEquals("testv1", kylinConfigExt.get("testk1"));
Assert.assertEquals("test v2", kylinConfigExt.get("tes tk2"));
Assert.assertEquals("t estv3", kylinConfigExt.get("tes tk3"));
+ Assert.assertEquals("10", kylinConfigExt.get("kylin.job.max-concurrent-jobs"));
+
+ try {
+ Map<String, String> testOverrideWithIllegalValue = Maps.newLinkedHashMap();
+ testOverrideWithIllegalValue.put("kylin.query.convert-sum-expression-enabled", "true");
+ testOverrideWithIllegalValue.put("kylin.job.max-concurrent-jobs", "test");
+ projectService.updateProjectConfig(PROJECT, testOverrideWithIllegalValue);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof KylinException);
+ Assert.assertTrue(e.getMessage().contains("must be a positive number"));
+ }
+
+ try {
+ Map<String, String> testOverrideWithOutofRangeValue = Maps.newLinkedHashMap();
+ testOverrideWithOutofRangeValue.put("kylin.query.convert-sum-expression-enabled", "true");
+ testOverrideWithOutofRangeValue.put("kylin.job.max-concurrent-jobs", "-1");
+ projectService.updateProjectConfig(PROJECT, testOverrideWithOutofRangeValue);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof KylinException);
+ Assert.assertTrue(e.getMessage().contains("must be a positive number"));
+ }
+
+
}
@Test
(kylin) 08/15: KYLIN-5740 Equivalent aggregations matches index failed
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0e3354414d89c4b4b81d0ec1db8d39705e760a40
Author: huangsheng <hu...@163.com>
AuthorDate: Fri Jun 30 12:10:57 2023 +0800
KYLIN-5740 Equivalent aggregations matches index failed
Equivalent aggregations matches index failed when using
the strategy of dimension as measure.
---
.../kylin/rest/service/AccessServiceTest.java | 4 +-
.../kylin/rest/service/AclTCRServiceTest.java | 2 +-
.../metadata/cube/cuboid/AggIndexMatcher.java | 13 ++-
.../metadata/project/NProjectManagerTest.java | 2 +-
.../kylin/metrics/HdfsCapacityMetricsTest.java | 2 +-
.../localmeta/data/DEFAULT.TEST_DIM_AS_MEASURE.csv | 3 +
.../metadata/_global/project/dim_as_measure.json | 6 ++
.../f29d1a92-c115-ec68-5575-9cfcc0d65890.json | 63 ++++++++++++++
.../4409fb82-8f3a-ba01-9852-0b05f1fe690f.json | 64 ++++++++++++++
.../f29d1a92-c115-ec68-5575-9cfcc0d65890.json | 64 ++++++++++++++
.../f29d1a92-c115-ec68-5575-9cfcc0d65890.json | 97 ++++++++++++++++++++++
.../table/DEFAULT.TEST_DIM_AS_MEASURE.json | 49 +++++++++++
.../newten/table/DEFAULT.TEST_DIM_AS_MEASURE.json | 49 +++++++++++
.../query/routing/QueryLayoutChooserTest.java | 47 ++++++++++-
.../kylin/rest/service/ProjectServiceTest.java | 6 +-
.../rest/service/QueryHistoryServiceTest.java | 2 +-
.../kylin/query/engine/AsyncQueryJobTest.java | 2 +-
17 files changed, 460 insertions(+), 15 deletions(-)
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
index f5402d462c..769f1dd326 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
@@ -606,14 +606,14 @@ public class AccessServiceTest extends NLocalFileMetadataTestCase {
@Test
public void testGetGrantedProjectsOfUser() throws IOException {
List<String> result = accessService.getGrantedProjectsOfUser("ADMIN");
- assertEquals(29, result.size());
+ assertEquals(30, result.size());
}
@Test
public void testGetGrantedProjectsOfUserOrGroup() throws IOException {
// admin user
List<String> result = accessService.getGrantedProjectsOfUserOrGroup("ADMIN", true);
- assertEquals(29, result.size());
+ assertEquals(30, result.size());
// normal user
result = accessService.getGrantedProjectsOfUserOrGroup("ANALYST", true);
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
index 803019720d..9af9f92b14 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
@@ -1316,7 +1316,7 @@ public class AclTCRServiceTest extends NLocalFileMetadataTestCase {
Mockito.when(userService.isGlobalAdmin("ADMIN")).thenReturn(true);
List<SidPermissionWithAclResponse> responses = accessService.getUserOrGroupAclPermissions(projects, "ADMIN",
true);
- Assert.assertEquals(29, responses.size());
+ Assert.assertEquals(30, responses.size());
Assert.assertTrue(responses.stream().allMatch(response -> "ADMIN".equals(response.getProjectPermission())));
// test normal group
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
index 5561581d72..a7efb17286 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
@@ -89,7 +89,7 @@ public class AggIndexMatcher extends IndexMatcher {
return new MatchResult();
}
log.trace("Matching agg index");
- Set<FunctionDesc> unmatchedMetrics = Sets.newHashSet(sqlDigest.aggregations);
+ Collection<FunctionDesc> unmatchedMetrics = Lists.newArrayList(sqlDigest.aggregations);
Set<Integer> unmatchedCols = initUnmatchedColumnIds(layout);
final Map<Integer, DeriveInfo> needDerive = Maps.newHashMap();
goThruDerivedDims(layout.getIndex(), needDerive, unmatchedCols);
@@ -140,9 +140,8 @@ public class AggIndexMatcher extends IndexMatcher {
functionDescs.addAll(cuboidLayout.getStreamingMeasures().values());
}
functionDescs.addAll(cuboidLayout.getOrderedMeasures().values());
-
for (MeasureDesc measureDesc : functionDescs) {
- aggregations.remove(measureDesc.getFunction());
+ aggregations.removeIf(functionDesc -> measureDesc.getFunction().equals(functionDesc));
}
}
@@ -154,8 +153,15 @@ public class AggIndexMatcher extends IndexMatcher {
Map<Integer, DeriveInfo> needDeriveCollector, List<CapabilityResult.CapabilityInfluence> influences) {
IndexEntity indexEntity = layoutEntity.getIndex();
Iterator<FunctionDesc> it = unmatchedAggs.iterator();
+ Set<FunctionDesc> matchedAggs = Sets.newHashSet();
while (it.hasNext()) {
FunctionDesc functionDesc = it.next();
+ if (matchedAggs.contains(functionDesc)) {
+ influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
+ it.remove();
+ continue;
+ }
+
if (functionDesc.isCountConstant()) {
it.remove();
continue;
@@ -180,6 +186,7 @@ public class AggIndexMatcher extends IndexMatcher {
if (FunctionDesc.DIMENSION_AS_MEASURES.contains(functionDesc.getExpression())) {
influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
+ matchedAggs.add(functionDesc);
it.remove();
}
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
index f3ed44b83f..4cfb808f39 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
@@ -68,7 +68,7 @@ public class NProjectManagerTest extends NLocalFileMetadataTestCase {
}
val projects = projectManager.listAllProjects();
- Assert.assertEquals(29, projects.size());
+ Assert.assertEquals(30, projects.size());
Assert.assertTrue(projects.stream().noneMatch(p -> p.getName().equals("test")));
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
index 28a8c5fada..500596429a 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
@@ -90,7 +90,7 @@ public class HdfsCapacityMetricsTest extends NLocalFileMetadataTestCase {
}
Assert.assertTrue(hdfsCapacityMetrics.getWorkingDirCapacity().isEmpty());
hdfsCapacityMetrics.writeHdfsMetrics();
- Assert.assertEquals(29, hdfsCapacityMetrics.getWorkingDirCapacity().size());
+ Assert.assertEquals(30, hdfsCapacityMetrics.getWorkingDirCapacity().size());
}
diff --git a/src/examples/test_case_data/localmeta/data/DEFAULT.TEST_DIM_AS_MEASURE.csv b/src/examples/test_case_data/localmeta/data/DEFAULT.TEST_DIM_AS_MEASURE.csv
new file mode 100644
index 0000000000..1a1f4e5686
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/data/DEFAULT.TEST_DIM_AS_MEASURE.csv
@@ -0,0 +1,3 @@
+22Q1,150.12,dongqu,huadongqu
+22Q2,200.59,zhongqu,huazhongqu
+22Q3,301.55,shanghaiqu,huadongqu
diff --git a/src/examples/test_case_data/localmeta/metadata/_global/project/dim_as_measure.json b/src/examples/test_case_data/localmeta/metadata/_global/project/dim_as_measure.json
new file mode 100644
index 0000000000..942a8244d5
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/_global/project/dim_as_measure.json
@@ -0,0 +1,6 @@
+{
+ "uuid": "c3c755b5-a164-9ae0-e1b2-9a296c3cb5a9",
+ "override_kylin_properties": {
+ "kylin.query.slowquery-detect-interval": "4"
+ }
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/dim_as_measure/dataflow/f29d1a92-c115-ec68-5575-9cfcc0d65890.json b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/dataflow/f29d1a92-c115-ec68-5575-9cfcc0d65890.json
new file mode 100644
index 0000000000..e309636c02
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/dataflow/f29d1a92-c115-ec68-5575-9cfcc0d65890.json
@@ -0,0 +1,63 @@
+{
+ "uuid" : "f29d1a92-c115-ec68-5575-9cfcc0d65890",
+ "last_modified" : 1688008789459,
+ "create_time" : 1688008685970,
+ "version" : "4.0.0.0",
+ "status" : "ONLINE",
+ "last_status" : null,
+ "cost" : 50,
+ "query_hit_count" : 0,
+ "last_query_time" : 0,
+ "layout_query_hit_count" : { },
+ "segments" : [ {
+ "id" : "4409fb82-8f3a-ba01-9852-0b05f1fe690f",
+ "name" : "FULL_BUILD",
+ "create_time_utc" : 1688008750346,
+ "status" : "READY",
+ "segRange" : {
+ "@class" : "org.apache.kylin.metadata.model.SegmentRange$TimePartitionedSegmentRange",
+ "date_range_start" : 0,
+ "date_range_end" : 9223372036854775807
+ },
+ "timeRange" : null,
+ "dimension_range_info_map" : {
+ "0" : {
+ "min" : "huadongqu",
+ "max" : "huazhongqu"
+ },
+ "1" : {
+ "min" : "shanghaiqu",
+ "max" : "zhongqu"
+ },
+ "2" : {
+ "min" : "150.1200",
+ "max" : "301.5500"
+ },
+ "3" : {
+ "min" : "22Q1",
+ "max" : "22Q3"
+ }
+ },
+ "parameters" : null,
+ "dictionaries" : null,
+ "snapshots" : null,
+ "last_build_time" : 1688008489488,
+ "source_count" : 3,
+ "source_bytes_size" : 90,
+ "column_source_bytes" : {
+ "DEFAULT.TEST_DIM_AS_MEASURE.REGION_NAME" : 27,
+ "DEFAULT.TEST_DIM_AS_MEASURE.BIG_REGION_NAME" : 21,
+ "DEFAULT.TEST_DIM_AS_MEASURE.ARRIVAL_TRANSIT_IN_PRM_AMT" : 24,
+ "DEFAULT.TEST_DIM_AS_MEASURE.PRO_MT_SEASON" : 12
+ },
+ "ori_snapshot_size" : { },
+ "additionalInfo" : { },
+ "is_realtime_segment" : false,
+ "is_snapshot_ready" : false,
+ "is_dict_ready" : false,
+ "is_flat_table_ready" : true,
+ "is_fact_view_ready" : false,
+ "multi_partitions" : [ ],
+ "max_bucket_id" : -1
+ } ]
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/dim_as_measure/dataflow_details/f29d1a92-c115-ec68-5575-9cfcc0d65890/4409fb82-8f3a-ba01-9852-0b05f1fe690f.json b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/dataflow_details/f29d1a92-c115-ec68-5575-9cfcc0d65890/4409fb82-8f3a-ba01-9852-0b05f1fe690f.json
new file mode 100644
index 0000000000..d917a6c151
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/dataflow_details/f29d1a92-c115-ec68-5575-9cfcc0d65890/4409fb82-8f3a-ba01-9852-0b05f1fe690f.json
@@ -0,0 +1,64 @@
+{
+ "uuid" : "4409fb82-8f3a-ba01-9852-0b05f1fe690f",
+ "last_modified" : 1688008750348,
+ "create_time" : 1688008750347,
+ "version" : "4.0.0.0",
+ "dataflow" : "f29d1a92-c115-ec68-5575-9cfcc0d65890",
+ "layout_instances" : [ {
+ "layout_id" : 180001,
+ "build_job_id" : "a6c09d79-a397-74fc-bb64-0d42c3c6ec0f-f29d1a92-c115-ec68-5575-9cfcc0d65890",
+ "rows" : 3,
+ "byte_size" : 1200,
+ "file_count" : 2,
+ "source_rows" : 3,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1688008777804,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ }, {
+ "layout_id" : 20000000001,
+ "build_job_id" : "a6c09d79-a397-74fc-bb64-0d42c3c6ec0f-f29d1a92-c115-ec68-5575-9cfcc0d65890",
+ "rows" : 3,
+ "byte_size" : 1438,
+ "file_count" : 2,
+ "source_rows" : 3,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1688008783698,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ }, {
+ "layout_id" : 190001,
+ "build_job_id" : "a6c09d79-a397-74fc-bb64-0d42c3c6ec0f-f29d1a92-c115-ec68-5575-9cfcc0d65890",
+ "rows" : 2,
+ "byte_size" : 844,
+ "file_count" : 2,
+ "source_rows" : 3,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1688008783698,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ }, {
+ "layout_id" : 170001,
+ "build_job_id" : "a6c09d79-a397-74fc-bb64-0d42c3c6ec0f-f29d1a92-c115-ec68-5575-9cfcc0d65890",
+ "rows" : 3,
+ "byte_size" : 871,
+ "file_count" : 2,
+ "source_rows" : 3,
+ "source_byte_size" : 0,
+ "partition_num" : 1,
+ "partition_values" : [ ],
+ "is_ready" : false,
+ "create_time" : 1688008783698,
+ "multi_partition" : [ ],
+ "abnormal_type" : null
+ } ]
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/dim_as_measure/index_plan/f29d1a92-c115-ec68-5575-9cfcc0d65890.json b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/index_plan/f29d1a92-c115-ec68-5575-9cfcc0d65890.json
new file mode 100644
index 0000000000..bddd047eb8
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/index_plan/f29d1a92-c115-ec68-5575-9cfcc0d65890.json
@@ -0,0 +1,64 @@
+{
+ "uuid" : "f29d1a92-c115-ec68-5575-9cfcc0d65890",
+ "last_modified" : 1688008749087,
+ "create_time" : 1687781017098,
+ "version" : "4.0.0.0",
+ "description" : null,
+ "rule_based_index" : {
+ "dimensions" : [ 0, 1 ],
+ "measures" : [ 100000 ],
+ "global_dim_cap" : null,
+ "aggregation_groups" : [ {
+ "includes" : [ 0, 1 ],
+ "measures" : [ 100000 ],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ ]
+ },
+ "index_range" : "EMPTY"
+ } ],
+ "layout_id_mapping" : [ 170001, 180001, 190001 ],
+ "parent_forward" : 3,
+ "index_start_id" : 170000,
+ "last_modify_time" : 1688008749085,
+ "layout_black_list" : [ ],
+ "scheduler_version" : 2,
+ "index_update_enabled" : true,
+ "base_layout_enabled" : true
+ },
+ "indexes" : [ {
+ "id" : 20000000000,
+ "dimensions" : [ 0, 1, 2, 3 ],
+ "measures" : [ ],
+ "layouts" : [ {
+ "id" : 20000000001,
+ "name" : null,
+ "owner" : null,
+ "col_order" : [ 0, 1, 2, 3 ],
+ "shard_by_columns" : [ ],
+ "partition_by_columns" : [ ],
+ "sort_by_columns" : [ ],
+ "storage_type" : 20,
+ "update_time" : 1687781017119,
+ "manual" : false,
+ "auto" : false,
+ "base" : true,
+ "draft_version" : null,
+ "index_range" : null
+ } ],
+ "next_layout_offset" : 2
+ } ],
+ "override_properties" : { },
+ "to_be_deleted_indexes" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 80,
+ "next_aggregation_index_id" : 200000,
+ "next_table_index_id" : 20000010000,
+ "agg_shard_by_columns" : [ ],
+ "extend_partition_columns" : [ ],
+ "layout_bucket_num" : { },
+ "approved_additional_recs" : 0,
+ "approved_removal_recs" : 0
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/dim_as_measure/model_desc/f29d1a92-c115-ec68-5575-9cfcc0d65890.json b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/model_desc/f29d1a92-c115-ec68-5575-9cfcc0d65890.json
new file mode 100644
index 0000000000..aff9008536
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/model_desc/f29d1a92-c115-ec68-5575-9cfcc0d65890.json
@@ -0,0 +1,97 @@
+{
+ "uuid" : "f29d1a92-c115-ec68-5575-9cfcc0d65890",
+ "last_modified" : 1688008696062,
+ "create_time" : 1687781016992,
+ "version" : "4.0.0.0",
+ "alias" : "ff_clone3",
+ "owner" : "ADMIN",
+ "config_last_modifier" : null,
+ "config_last_modified" : 0,
+ "description" : "",
+ "fact_table" : "DEFAULT.TEST_DIM_AS_MEASURE",
+ "fact_table_alias" : null,
+ "management_type" : "MODEL_BASED",
+ "join_tables" : [ ],
+ "filter_condition" : "",
+ "partition_desc" : null,
+ "capacity" : "MEDIUM",
+ "segment_config" : {
+ "auto_merge_enabled" : null,
+ "auto_merge_time_ranges" : null,
+ "volatile_range" : null,
+ "retention_range" : null,
+ "create_empty_segment_enabled" : false
+ },
+ "data_check_desc" : null,
+ "semantic_version" : 0,
+ "storage_type" : 0,
+ "model_type" : "BATCH",
+ "all_named_columns" : [ {
+ "id" : 0,
+ "name" : "REGION_NAME",
+ "column" : "TEST_DIM_AS_MEASURE.REGION_NAME",
+ "status" : "DIMENSION"
+ }, {
+ "id" : 1,
+ "name" : "BIG_REGION_NAME",
+ "column" : "TEST_DIM_AS_MEASURE.BIG_REGION_NAME",
+ "status" : "DIMENSION"
+ }, {
+ "id" : 2,
+ "name" : "ARRIVAL_TRANSIT_IN_PRM_AMT",
+ "column" : "TEST_DIM_AS_MEASURE.ARRIVAL_TRANSIT_IN_PRM_AMT",
+ "status" : "DIMENSION"
+ }, {
+ "id" : 3,
+ "name" : "PRO_MT_SEASON",
+ "column" : "TEST_DIM_AS_MEASURE.PRO_MT_SEASON",
+ "status" : "DIMENSION"
+ } ],
+ "all_measures" : [ {
+ "name" : "COUNT_ALL",
+ "function" : {
+ "expression" : "COUNT",
+ "parameters" : [ {
+ "type" : "constant",
+ "value" : "1"
+ } ],
+ "returntype" : "bigint"
+ },
+ "column" : null,
+ "comment" : null,
+ "id" : 100000,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ }, {
+ "name" : "BIG_REGION_NAME",
+ "function" : {
+ "expression" : "MAX",
+ "parameters" : [ {
+ "type" : "column",
+ "value" : "TEST_DIM_AS_MEASURE.BIG_REGION_NAME"
+ } ],
+ "returntype" : "varchar(4096)"
+ },
+ "column" : null,
+ "comment" : "",
+ "id" : 100001,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ } ],
+ "recommendations_count" : 0,
+ "computed_columns" : [ ],
+ "canvas" : {
+ "coordinate" : {
+ "TEST_DIM_AS_MEASURE" : {
+ "x" : 594.3035888671874,
+ "y" : 224.26886664496536,
+ "width" : 200.0,
+ "height" : 487.7777777777778
+ }
+ },
+ "zoom" : 9.0
+ },
+ "multi_partition_desc" : null,
+ "multi_partition_key_mapping" : null,
+ "fusion_id" : null
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/dim_as_measure/table/DEFAULT.TEST_DIM_AS_MEASURE.json b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/table/DEFAULT.TEST_DIM_AS_MEASURE.json
new file mode 100644
index 0000000000..acd6f10a5b
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/dim_as_measure/table/DEFAULT.TEST_DIM_AS_MEASURE.json
@@ -0,0 +1,49 @@
+{
+ "uuid" : "fef38e84-8385-a0e8-3fc3-b515aabbe461",
+ "last_modified" : 0,
+ "create_time" : 1687780438849,
+ "version" : "4.0.0.0",
+ "name" : "TEST_DIM_AS_MEASURE",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "PRO_MT_SEASON",
+ "datatype" : "varchar(4096)",
+ "case_sensitive_name" : "pro_mt_season"
+ }, {
+ "id" : "2",
+ "name" : "ARRIVAL_TRANSIT_IN_PRM_AMT",
+ "datatype" : "decimal(18,4)",
+ "case_sensitive_name" : "arrival_transit_in_prm_amt"
+ }, {
+ "id" : "3",
+ "name" : "BIG_REGION_NAME",
+ "datatype" : "varchar(4096)",
+ "case_sensitive_name" : "big_region_name"
+ }, {
+ "id" : "4",
+ "name" : "REGION_NAME",
+ "datatype" : "varchar(4096)",
+ "case_sensitive_name" : "region_name"
+ } ],
+ "source_type" : 9,
+ "table_type" : "EXTERNAL",
+ "top" : false,
+ "increment_loading" : false,
+ "last_snapshot_path" : null,
+ "last_snapshot_size" : 0,
+ "snapshot_last_modified" : 0,
+ "query_hit_count" : 0,
+ "partition_column" : null,
+ "snapshot_partitions" : { },
+ "snapshot_partitions_info" : { },
+ "snapshot_total_rows" : 0,
+ "snapshot_partition_col" : null,
+ "selected_snapshot_partition_col" : null,
+ "temp_snapshot_path" : null,
+ "snapshot_has_broken" : false,
+ "database" : "default",
+ "transactional" : false,
+ "rangePartition" : false,
+ "partition_desc" : null,
+ "table_comment" : null
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/newten/table/DEFAULT.TEST_DIM_AS_MEASURE.json b/src/examples/test_case_data/localmeta/metadata/newten/table/DEFAULT.TEST_DIM_AS_MEASURE.json
new file mode 100644
index 0000000000..acd6f10a5b
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/newten/table/DEFAULT.TEST_DIM_AS_MEASURE.json
@@ -0,0 +1,49 @@
+{
+ "uuid" : "fef38e84-8385-a0e8-3fc3-b515aabbe461",
+ "last_modified" : 0,
+ "create_time" : 1687780438849,
+ "version" : "4.0.0.0",
+ "name" : "TEST_DIM_AS_MEASURE",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "PRO_MT_SEASON",
+ "datatype" : "varchar(4096)",
+ "case_sensitive_name" : "pro_mt_season"
+ }, {
+ "id" : "2",
+ "name" : "ARRIVAL_TRANSIT_IN_PRM_AMT",
+ "datatype" : "decimal(18,4)",
+ "case_sensitive_name" : "arrival_transit_in_prm_amt"
+ }, {
+ "id" : "3",
+ "name" : "BIG_REGION_NAME",
+ "datatype" : "varchar(4096)",
+ "case_sensitive_name" : "big_region_name"
+ }, {
+ "id" : "4",
+ "name" : "REGION_NAME",
+ "datatype" : "varchar(4096)",
+ "case_sensitive_name" : "region_name"
+ } ],
+ "source_type" : 9,
+ "table_type" : "EXTERNAL",
+ "top" : false,
+ "increment_loading" : false,
+ "last_snapshot_path" : null,
+ "last_snapshot_size" : 0,
+ "snapshot_last_modified" : 0,
+ "query_hit_count" : 0,
+ "partition_column" : null,
+ "snapshot_partitions" : { },
+ "snapshot_partitions_info" : { },
+ "snapshot_total_rows" : 0,
+ "snapshot_partition_col" : null,
+ "selected_snapshot_partition_col" : null,
+ "temp_snapshot_path" : null,
+ "snapshot_has_broken" : false,
+ "database" : "default",
+ "transactional" : false,
+ "rangePartition" : false,
+ "partition_desc" : null,
+ "table_comment" : null
+}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutChooserTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutChooserTest.java
index 4a47b4db74..7f9408788d 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutChooserTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/QueryLayoutChooserTest.java
@@ -43,6 +43,7 @@ import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
+import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.util.MetadataTestUtils;
import org.apache.kylin.util.OlapContextTestUtil;
@@ -461,9 +462,9 @@ public class QueryLayoutChooserTest extends NLocalWithSparkSessionTest {
olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
// hit layout 1010002
- // 1. shardby layout are has higher priority over non-shardby layout,
+ // 1. shardby layout are has higher priority over non-shardby layout,
// so 1010001 is skipped, although it has a better dim order
- // 2. trans_id has a higher cardinality, so 1010002 with shard on trans_id
+ // 2. trans_id has a higher cardinality, so 1010002 with shard on trans_id
// is preferred over 1010003 with shard on cal_dt
NLayoutCandidate layoutCandidate = QueryLayoutChooser.selectLayoutCandidate(dataflow,
dataflow.getQueryableSegments(), olapContext.getSQLDigest(), null);
@@ -749,6 +750,48 @@ public class QueryLayoutChooserTest extends NLocalWithSparkSessionTest {
}
}
+ /**
+ * Start with an empty project to get the OlapContexts not pollute is very important.
+ */
+ @Test
+ public void testMatchDimAsMeasure() throws SqlParseException {
+ String emptyProject = "newten";
+ overwriteSystemProp("kylin.query.use-tableindex-answer-non-raw-query", "true");
+ String project = "dim_as_measure";
+ String modelId = "f29d1a92-c115-ec68-5575-9cfcc0d65890";
+ NDataflow dataflow = NDataflowManager.getInstance(getTestConfig(), project).getDataflow(modelId);
+
+ {
+ String sql = "select max(BIG_REGION_NAME) max1, max(case when 1=1 then BIG_REGION_NAME end) max2 from TEST_DIM_AS_MEASURE";
+ OLAPContext olapContext = OlapContextTestUtil.getOlapContexts(emptyProject, sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap = OlapContextTestUtil.matchJoins(dataflow.getModel(),
+ olapContext);
+ olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
+ NLayoutCandidate layoutCandidate = QueryLayoutChooser.selectLayoutCandidate(dataflow,
+ dataflow.getQueryableSegments(), olapContext.getSQLDigest(), null);
+ Assert.assertNotNull(layoutCandidate);
+ Assert.assertFalse(layoutCandidate.getLayoutEntity().getIndex().isTableIndex());
+ for (CapabilityResult.CapabilityInfluence inf : layoutCandidate.getCapabilityResult().influences) {
+ Assert.assertTrue(inf instanceof CapabilityResult.DimensionAsMeasure);
+ }
+ }
+
+ {
+ String sql = "select max(REGION_NAME) max1, max(case when 1=1 then BIG_REGION_NAME end) max2 from TEST_DIM_AS_MEASURE";
+ OLAPContext olapContext = OlapContextTestUtil.getOlapContexts(emptyProject, sql).get(0);
+ Map<String, String> sqlAlias2ModelNameMap = OlapContextTestUtil.matchJoins(dataflow.getModel(),
+ olapContext);
+ olapContext.fixModel(dataflow.getModel(), sqlAlias2ModelNameMap);
+ NLayoutCandidate layoutCandidate = QueryLayoutChooser.selectLayoutCandidate(dataflow,
+ dataflow.getQueryableSegments(), olapContext.getSQLDigest(), null);
+ Assert.assertNotNull(layoutCandidate);
+ Assert.assertFalse(layoutCandidate.getLayoutEntity().getIndex().isTableIndex());
+ for (CapabilityResult.CapabilityInfluence inf : layoutCandidate.getCapabilityResult().influences) {
+ Assert.assertTrue(inf instanceof CapabilityResult.DimensionAsMeasure);
+ }
+ }
+ }
+
@Test
public void testMatchJoinWithEnhancedMode() throws SqlParseException {
getTestConfig().setProperty("kylin.query.join-match-optimization-enabled", "true");
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
index f47a64020b..054755f307 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
@@ -235,14 +235,14 @@ public class ProjectServiceTest extends NLocalFileMetadataTestCase {
public void testGetReadableProjects() {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances = projectService.getReadableProjects("", false);
- Assert.assertEquals(29, projectInstances.size());
+ Assert.assertEquals(30, projectInstances.size());
}
@Test
public void testGetAdminProjects() throws Exception {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances = projectService.getAdminProjects();
- Assert.assertEquals(29, projectInstances.size());
+ Assert.assertEquals(30, projectInstances.size());
}
@Test
@@ -256,7 +256,7 @@ public class ProjectServiceTest extends NLocalFileMetadataTestCase {
public void testGetReadableProjectsHasNoPermissionProject() {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances = projectService.getReadableProjects("", false);
- Assert.assertEquals(29, projectInstances.size());
+ Assert.assertEquals(30, projectInstances.size());
}
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
index 67c2e6c03d..3f2f3cf692 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
@@ -364,7 +364,7 @@ public class QueryHistoryServiceTest extends NLocalFileMetadataTestCase {
// get all tables
tableMap = queryHistoryService.getQueryHistoryTableMap(null);
- Assert.assertEquals(29, tableMap.size());
+ Assert.assertEquals(30, tableMap.size());
// not existing project
tableMap = queryHistoryService.getQueryHistoryTableMap(Lists.newArrayList("not_existing_project"));
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java
index 4ba275bf48..4c57c0d5eb 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java
@@ -218,7 +218,7 @@ public class AsyncQueryJobTest extends NLocalFileMetadataTestCase {
rawResourceMap.put(zipEntry.getName(), raw);
}
}
- Assert.assertEquals(85, rawResourceMap.size());
+ Assert.assertEquals(86, rawResourceMap.size());
}
private void testKylinConfig(FileSystem workingFileSystem, FileStatus metaFileStatus) throws IOException {
(kylin) 04/15: KYLIN-5760 Support matching model exactly for list jobs
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b15367e3e2682b195966f5b59dbf09f8867a792e
Author: Hang Jia <75...@qq.com>
AuthorDate: Thu Jun 29 14:24:12 2023 +0800
KYLIN-5760 Support matching model exactly for list jobs
---
.../kap/secondstorage/metadata/TableFlow.java | 13 ++++++
.../kylin/rest/controller/JobController.java | 4 +-
.../kylin/rest/controller/v2/JobControllerV2.java | 2 +-
.../kylin/rest/controller/JobControllerTest.java | 6 +--
.../rest/controller/v2/JobControllerV2Test.java | 11 ++---
.../org/apache/kylin/rest/request/JobFilter.java | 2 +
.../org/apache/kylin/rest/service/JobService.java | 5 +++
.../apache/kylin/rest/service/JobServiceTest.java | 49 ++++++++++++++++++----
.../apache/kylin/rest/service/ModelService.java | 9 ++--
9 files changed, 77 insertions(+), 24 deletions(-)
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java
index a4add8edca..908f591e13 100644
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java
+++ b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java
@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import com.fasterxml.jackson.annotation.JsonManagedReference;
@@ -72,6 +73,8 @@ public class TableFlow extends RootPersistentEntity
return new Builder();
}
+ public static final String TABLEFLOW_RESOURCE_ROOT = "/clickhouse_data";
+
protected transient Manager<TableFlow> manager;
@Override
public void setManager(Manager<TableFlow> manager) {
@@ -171,6 +174,16 @@ public class TableFlow extends RootPersistentEntity
});
}
+ @Override
+ public String getResourcePath() {
+ return concatResourcePath(getUuid(), manager.project);
+ }
+
+ public static String concatResourcePath(String name, String project) {
+ return new StringBuilder().append("/").append(project).append(TABLEFLOW_RESOURCE_ROOT)
+ .append("/").append(name).append(MetadataConstants.FILE_SURFIX).toString();
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
index ea3e921baa..e47d01b658 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -94,6 +94,7 @@ public class JobController extends BaseController {
@RequestParam(value = "time_filter") Integer timeFilter,
@RequestParam(value = "subject", required = false) String subject,
@RequestParam(value = "key", required = false) String key,
+ @RequestParam(value = "exact", required = false, defaultValue = "false") boolean exactMatch,
@RequestParam(value = "project", required = false) String project,
@RequestParam(value = "page_offset", required = false, defaultValue = "0") Integer pageOffset,
@RequestParam(value = "page_size", required = false, defaultValue = "10") Integer pageSize,
@@ -101,7 +102,8 @@ public class JobController extends BaseController {
@RequestParam(value = "reverse", required = false, defaultValue = "true") boolean reverse) {
jobService.checkJobStatus(statuses);
checkRequiredArg("time_filter", timeFilter);
- JobFilter jobFilter = new JobFilter(statuses, jobNames, timeFilter, subject, key, project, sortBy, reverse);
+ JobFilter jobFilter = new JobFilter(statuses, jobNames, timeFilter, subject, key, exactMatch, project, sortBy,
+ reverse);
DataResult<List<ExecutableResponse>> executables;
if (!StringUtils.isEmpty(project)) {
executables = jobService.listJobs(jobFilter, pageOffset, pageSize);
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java
index 720238fb53..6844578785 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java
@@ -108,7 +108,7 @@ public class JobControllerV2 extends BaseController {
JobFilter jobFilter = new JobFilter(statuses,
Objects.isNull(jobName) ? Lists.newArrayList() : Lists.newArrayList(jobName), timeFilter, null, key,
- project, sortBy, reverse);
+ false, project, sortBy, reverse);
List<ExecutableResponse> executables = null;
if (!StringUtils.isEmpty(project)) {
executables = jobService.listJobs(jobFilter);
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
index 1cc841cd8e..560bf6ee1f 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/JobControllerTest.java
@@ -92,7 +92,7 @@ public class JobControllerTest extends NLocalFileMetadataTestCase {
List<ExecutableResponse> jobs = new ArrayList<>();
List<String> jobNames = Lists.newArrayList();
List<String> statuses = Lists.newArrayList("NEW", "RUNNING");
- JobFilter jobFilter = new JobFilter(statuses, jobNames, 4, "", "", "default", "job_name", false);
+ JobFilter jobFilter = new JobFilter(statuses, jobNames, 4, "", "", false, "default", "job_name", false);
Mockito.when(jobService.listJobs(jobFilter)).thenReturn(jobs);
mockMvc.perform(MockMvcRequestBuilders.get("/api/jobs").contentType(MediaType.APPLICATION_JSON)
.param("project", "default").param("page_offset", "0").param("page_size", "10")
@@ -100,8 +100,8 @@ public class JobControllerTest extends NLocalFileMetadataTestCase {
.param("statuses", "NEW,RUNNING").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
- Mockito.verify(jobController).getJobList(statuses, jobNames, 1, "", "", "default", 0, 10, "last_modified",
- true);
+ Mockito.verify(jobController).getJobList(statuses, jobNames, 1, "", "", false, "default", 0, 10,
+ "last_modified", true);
}
@Test
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java
index eec40babd4..ad66615c13 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java
@@ -107,8 +107,8 @@ public class JobControllerV2Test extends NLocalFileMetadataTestCase {
public void testGetJobs() throws Exception {
List<ExecutableResponse> jobs = new ArrayList<>();
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList("NEW"), jobNames, 4, "", "", "default", "job_name",
- false);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList("NEW"), jobNames, 4, "", "", false, "default",
+ "job_name", false);
Mockito.when(jobService.listJobs(jobFilter)).thenReturn(jobs);
mockMvc.perform(MockMvcRequestBuilders.get("/api/jobs").contentType(MediaType.APPLICATION_JSON)
.param("projectName", "default").param("pageOffset", "0").param("pageSize", "10")
@@ -124,7 +124,8 @@ public class JobControllerV2Test extends NLocalFileMetadataTestCase {
public void testGetJobsWithoutProjectAndSortby() throws Exception {
List<ExecutableResponse> jobs = new ArrayList<>();
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, null, null, null, "job_name", true);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, null, null, false, null, "job_name",
+ true);
Mockito.when(jobService.listGlobalJobs(jobFilter, 0, Integer.MAX_VALUE))
.thenReturn(new DataResult<>(jobs, 0, 0, 0));
mockMvc.perform(
@@ -171,8 +172,8 @@ public class JobControllerV2Test extends NLocalFileMetadataTestCase {
public void testGetJobsException_pageOffset_pageSize() throws Exception {
List<ExecutableResponse> jobs = new ArrayList<>();
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList("NEW"), jobNames, 4, "", "", "default", "job_name",
- false);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList("NEW"), jobNames, 4, "", "", false, "default",
+ "job_name", false);
Mockito.when(jobService.listJobs(jobFilter)).thenReturn(jobs);
mockMvc.perform(MockMvcRequestBuilders.get("/api/jobs").contentType(MediaType.APPLICATION_JSON)
.param("projectName", "default").param("pageOffset", "a").param("pageSize", "10")
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/request/JobFilter.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/request/JobFilter.java
index 05f8618b43..d184b15fc4 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/request/JobFilter.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/request/JobFilter.java
@@ -39,6 +39,8 @@ public class JobFilter {
private String key;
+ private boolean exactMatch;
+
private String project;
private String sortBy;
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
index 71d18e1771..212f9f4365 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -257,6 +257,11 @@ public class JobService extends BasicService implements JobSupporter, ISmartAppl
modelManagerMaps.put(executablePO.getProject(), modelManager);
}
return createExecutablePOSortBean(executablePO, modelManager);
+ }).filter(executablePO -> {
+ if (!jobFilter.isExactMatch() || StringUtils.isEmpty(jobFilter.getKey())) {
+ return true;
+ }
+ return jobFilter.getKey().equalsIgnoreCase(executablePO.getTargetSubject());
}).sorted(comparator).collect(Collectors.toList());
}
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index 7e076453fc..5a98b308b9 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -265,7 +265,7 @@ public class JobServiceTest extends LogOutputTestCase {
getTestConfig().setProperty("kylin.streaming.enabled", "false");
// test size
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", "default", "", true);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", false, "default", "", true);
List<ExecutableResponse> jobs = jobService.listJobs(jobFilter);
Assert.assertEquals(3, jobs.size());
jobService.addOldParams(jobs);
@@ -399,8 +399,8 @@ public class JobServiceTest extends LogOutputTestCase {
Mockito.when(executableDao.getJobs(Mockito.anyLong(), Mockito.anyLong())).thenReturn(mockJobs);
{
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 0, "", "", "default", "total_duration",
- true);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 0, "", "", false, "default",
+ "total_duration", true);
List<ExecutableResponse> jobs = jobService.listJobs(jobFilter);
val totalDurationArrays = jobs.stream().map(ExecutableResponse::getTotalDuration)
@@ -419,11 +419,41 @@ public class JobServiceTest extends LogOutputTestCase {
}
}
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 0, "", "default", "default", "", false);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 0, "", "default", false, "default", "",
+ false);
List<ExecutableResponse> jobs = jobService.listJobs(jobFilter);
Assert.assertEquals(2, jobs.size());
}
+ @Test
+ public void testFilterJobExactMatch() throws Exception {
+ NExecutableManager executableManager = NExecutableManager.getInstance(getTestConfig(), "default");
+ Mockito.when(jobService.getManager(NExecutableManager.class, "default")).thenReturn(executableManager);
+ ReflectionTestUtils.setField(executableManager, "executableDao", executableDao);
+ val mockJobs = mockDetailJobs(false);
+ Mockito.when(executableDao.getJobs(Mockito.anyLong(), Mockito.anyLong())).thenReturn(mockJobs);
+
+ for (int i = 0; i < 3; i++) {
+ if (i < 2) {
+ mockJobs.get(i).setJobType(JobTypeEnum.SECOND_STORAGE_NODE_CLEAN);
+ } else {
+ mockJobs.get(i).setJobType(JobTypeEnum.TABLE_SAMPLING);
+ }
+ }
+ List<String> jobNames = Lists.newArrayList();
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 0, "", "def", false, "default", "", false);
+ List<ExecutableResponse> jobs = jobService.listJobs(jobFilter);
+ Assert.assertEquals(2, jobs.size());
+
+ JobFilter jobFilter2 = new JobFilter(Lists.newArrayList(), jobNames, 0, "", "def", true, "default", "", false);
+ List<ExecutableResponse> jobs2 = jobService.listJobs(jobFilter2);
+ Assert.assertEquals(0, jobs2.size());
+
+ JobFilter jobFilter3 = new JobFilter(Lists.newArrayList(), jobNames, 0, "", null, true, "default", "", false);
+ List<ExecutableResponse> jobs3 = jobService.listJobs(jobFilter3);
+ Assert.assertEquals(3, jobs3.size());
+ }
+
private List<ProjectInstance> mockProjects() {
ProjectInstance defaultProject = new ProjectInstance();
defaultProject.setName("default");
@@ -467,7 +497,7 @@ public class JobServiceTest extends LogOutputTestCase {
Mockito.when(executableManager1.getAllExecutables(Mockito.anyLong(), Mockito.anyLong())).thenReturn(mockJobs1);
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", "default", "", true);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", false, "default", "", true);
List<ExecutableResponse> jobs = jobService.listGlobalJobs(jobFilter, 0, 10).getValue();
Assert.assertEquals(4, jobs.size());
Assert.assertEquals("default1", jobs.get(3).getProject());
@@ -1107,7 +1137,7 @@ public class JobServiceTest extends LogOutputTestCase {
executable.setName("test_create_time");
manager.addJob(executable);
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", "default", "", true);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", false, "default", "", true);
List<ExecutableResponse> jobs = jobService.listJobs(jobFilter);
Assert.assertTrue(jobs.get(0).getCreateTime() > 0);
}
@@ -1125,7 +1155,7 @@ public class JobServiceTest extends LogOutputTestCase {
manager.addJob(job1);
manager.addJob(samplingJob);
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", "default", "", true);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", false, "default", "", true);
jobFilter.setSortBy("job_name");
List<ExecutableResponse> jobs = jobService.listJobs(jobFilter);
@@ -1300,7 +1330,8 @@ public class JobServiceTest extends LogOutputTestCase {
Mockito.when(manager.getAllJobs(Mockito.anyLong(), Mockito.anyLong()))
.thenReturn(Collections.singletonList(job1));
- JobFilter jobFilter = new JobFilter(Lists.newArrayList(), Lists.newArrayList(), 4, "", "", "default", "", true);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), Lists.newArrayList(), 4, "", "", false, "default", "",
+ true);
List<ExecutableResponse> jobs = jobService.listJobs(jobFilter);
Assert.assertEquals(1, jobs.size());
@@ -1959,7 +1990,7 @@ public class JobServiceTest extends LogOutputTestCase {
getTestConfig().setProperty("kylin.streaming.enabled", "false");
// test size
List<String> jobNames = Lists.newArrayList();
- JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", "default", "", true);
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", false, "default", "", true);
List<ExecutableResponse> jobs = jobService.listJobs(jobFilter);
List<ExecutableResponse> executableResponses = jobService.addOldParams(jobs);
ExecutableResponse executable = executableResponses.get(0);
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index b4b1721dee..1ad9ce6cd1 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -372,11 +372,10 @@ public class ModelService extends AbstractModelService implements TableModelSupp
private void addOldSegmentParams(NDataModel model, NDataModelOldParams oldParams,
List<AbstractExecutable> executables) {
- List<NDataSegmentResponse> segments = getSegmentsResponse(model.getId(), model.getProject(), "1",
- String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true);
- calculateRecordSizeAndCount(segments, oldParams);
-
- if (model instanceof NDataModelResponse) {
+ if (((model instanceof NDataModelResponse) && !(model instanceof NDataModelLiteResponse)) || model.isFusionModel()) {
+ List<NDataSegmentResponse> segments = getSegmentsResponse(model.getId(), model.getProject(), "1",
+ String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true);
+ calculateRecordSizeAndCount(segments, oldParams);
((NDataModelResponse) model).setSegments(segments);
((NDataModelResponse) model).setHasSegments(
((NDataModelResponse) model).isHasSegments() || CollectionUtils.isNotEmpty(segments));
(kylin) 13/15: KYLIN-5741 punctuation optimized
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 227e7a068360978133f8188cff859723cea458bf
Author: zhong.zhu <mi...@icloud.com>
AuthorDate: Tue Dec 19 11:52:18 2023 +0800
KYLIN-5741 punctuation optimized
---
.../org/apache/kylin/common/msg/CnMessage.java | 2 +-
.../kylin/rest/service/MetaStoreService.java | 24 ++--
.../org/apache/spark/sql/udf/CeilFloorTest.scala | 139 +++++++++++----------
.../apache/kylin/helper/MetadataToolHelper.java | 20 +--
4 files changed, 94 insertions(+), 91 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index ed626ec5d1..09c3925756 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -32,7 +32,7 @@ public class CnMessage extends Message {
private static final String TASK_TIMEOUT = "执行超时";
private static final String PARAMETER_EMPTY = "请输入参数 “%s” 的值。";
- private static final String PARAMETER_MUST_BE_POSITIVE_NUMBER= "参数 %s 的值必须为非负数.";
+ private static final String PARAMETER_MUST_BE_POSITIVE_NUMBER= "参数 %s 的值必须为非负数。";
protected CnMessage() {
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
index bf44347d69..4976131faa 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
@@ -75,6 +75,10 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MetadataChecker;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.helper.RoutineToolHelper;
import org.apache.kylin.metadata.cube.model.IndexEntity;
@@ -99,6 +103,10 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.util.QueryHisStoreUtil;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
+import org.apache.kylin.metadata.recommendation.candidate.RawRecItem;
+import org.apache.kylin.metadata.recommendation.candidate.RawRecManager;
+import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
import org.apache.kylin.metadata.view.LogicalView;
import org.apache.kylin.metadata.view.LogicalViewManager;
import org.apache.kylin.rest.aspect.Transaction;
@@ -121,16 +129,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
-
-import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
-import org.apache.kylin.metadata.recommendation.candidate.RawRecItem;
-import org.apache.kylin.metadata.recommendation.candidate.RawRecManager;
-import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
-
import lombok.Setter;
import lombok.val;
import lombok.var;
@@ -466,9 +464,9 @@ public class MetaStoreService extends BasicService {
if (config.isDDLLogicalViewEnabled() && missTableDesc.isLogicalView()) {
LogicalView logicalView = LogicalViewManager.getInstance(config).get(missTableDesc.getName());
if (logicalView != null && !targetProject.equalsIgnoreCase(logicalView.getCreatedProject())) {
- throw new KylinException(FAILED_CREATE_MODEL, String.format(Locale.ROOT,
- " Logical View %s can only add in project %s",
- missTableDesc.getName(), logicalView.getCreatedProject()));
+ throw new KylinException(FAILED_CREATE_MODEL,
+ String.format(Locale.ROOT, " Logical View %s can only add in project %s",
+ missTableDesc.getName(), logicalView.getCreatedProject()));
}
}
}
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/udf/CeilFloorTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/udf/CeilFloorTest.scala
index d0666ff906..b4a1ab06bd 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/udf/CeilFloorTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/udf/CeilFloorTest.scala
@@ -35,86 +35,91 @@ class CeilFloorTest extends SparderBaseFunSuite with SharedSparkSession with Bef
}
test("test ceil") {
- query("select ceil_datetime(date'2012-02-29', 'year')").equals("[2013-01-01 00:00:00.0]")
- query("select ceil_datetime(date'2012-02-29', 'quarter')").equals("[2012-04-01 00:00:00.0]")
- query("select ceil_datetime(date'2012-02-29', 'month')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime(date'2012-02-29', 'week')").equals("[2012-03-05 00:00:00.0]")
- query("select ceil_datetime(date'2012-02-29', 'DAY')").equals("[2012-02-29 00:00:00.0]")
- query("select ceil_datetime(date'2012-02-29', 'hour')").equals("[2012-02-29 00:00:00.0]")
- query("select ceil_datetime(date'2012-02-29', 'minute')").equals("[2012-02-29 00:00:00.0]")
- query("select ceil_datetime(date'2012-02-29', 'second')").equals("[2012-02-29 00:00:00.0]")
+ query("select ceil_datetime(date'2012-02-29', 'year')", "[2013-01-01 00:00:00.0]")
+ query("select ceil_datetime(date'2012-02-29', 'quarter')", "[2012-04-01 00:00:00.0]")
+ query("select ceil_datetime(date'2012-02-29', 'month')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime(date'2012-02-29', 'week')", "[2012-03-05 00:00:00.0]")
+ query("select ceil_datetime(date'2012-02-29', 'DAY')", "[2012-02-29 00:00:00.0]")
+ query("select ceil_datetime(date'2012-02-29', 'hour')", "[2012-02-29 00:00:00.0]")
+ query("select ceil_datetime(date'2012-02-29', 'minute')", "[2012-02-29 00:00:00.0]")
+ query("select ceil_datetime(date'2012-02-29', 'second')", "[2012-02-29 00:00:00.0]")
- query("select ceil_datetime('2012-02-29', 'year')").equals("[2013-01-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29', 'quarter')").equals("[2012-04-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29', 'month')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29', 'week')").equals("[2012-03-05 00:00:00.0]")
- query("select ceil_datetime('2012-02-29', 'DAY')").equals("[2012-02-29 00:00:00.0]")
- query("select ceil_datetime('2012-02-29', 'hour')").equals("[2012-02-29 00:00:00.0]")
- query("select ceil_datetime('2012-02-29', 'minute')").equals("[2012-02-29 00:00:00.0]")
- query("select ceil_datetime('2012-02-29', 'second')").equals("[2012-02-29 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29', 'year')", "[2013-01-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29', 'quarter')", "[2012-04-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29', 'month')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29', 'week')", "[2012-03-05 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29', 'DAY')", "[2012-02-29 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29', 'hour')", "[2012-02-29 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29', 'minute')", "[2012-02-29 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29', 'second')", "[2012-02-29 00:00:00.0]")
- query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'year')").equals("[2013-01-01 00:00:00.0]")
- query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'quarter')").equals("[2012-04-01 00:00:00.0]")
- query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'month')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'week')").equals("[2012-03-05 00:00:00.0]")
- query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'DAY')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'hour')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'minute')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'second')").equals("[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'year')", "[2013-01-01 00:00:00.0]")
+ query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'quarter')", "[2012-04-01 00:00:00.0]")
+ query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'month')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'week')", "[2012-03-05 00:00:00.0]")
+ query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'DAY')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'hour')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'minute')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime(timestamp'2012-02-29 23:59:59.1', 'second')", "[2012-03-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29 23:59:59.1', 'year')").equals("[2013-01-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29 23:59:59.1', 'quarter')").equals("[2012-04-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29 23:59:59.1', 'month')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29 23:59:59.1', 'week')").equals("[2012-03-05 00:00:00.0]")
- query("select ceil_datetime('2012-02-29 23:59:59.1', 'DAY')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29 23:59:59.1', 'hour')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29 23:59:59.1', 'minute')").equals("[2012-03-01 00:00:00.0]")
- query("select ceil_datetime('2012-02-29 23:59:59.1', 'second')").equals("[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29 23:59:59.1', 'year')", "[2013-01-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29 23:59:59.1', 'quarter')", "[2012-04-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29 23:59:59.1', 'month')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29 23:59:59.1', 'week')", "[2012-03-05 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29 23:59:59.1', 'DAY')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29 23:59:59.1', 'hour')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29 23:59:59.1', 'minute')", "[2012-03-01 00:00:00.0]")
+ query("select ceil_datetime('2012-02-29 23:59:59.1', 'second')", "[2012-03-01 00:00:00.0]")
}
test("test floor") {
- query("select floor_datetime(date'2012-02-29', 'year')").equals("[2013-01-01 00:00:00.0]")
- query("select floor_datetime(date'2012-02-29', 'quarter')").equals("[2012-04-01 00:00:00.0]")
- query("select floor_datetime(date'2012-02-29', 'month')").equals("[2012-03-01 00:00:00.0]")
- query("select floor_datetime(date'2012-02-29', 'week')").equals("[2012-03-05 00:00:00.0]")
- query("select floor_datetime(date'2012-02-29', 'DAY')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime(date'2012-02-29', 'hour')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime(date'2012-02-29', 'minute')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime(date'2012-02-29', 'second')").equals("[2012-02-29 00:00:00.0]")
+ query("select floor_datetime(date'2012-02-29', 'year')", "[2012-01-01 00:00:00.0]")
+ query("select floor_datetime(date'2012-02-29', 'quarter')", "[2012-01-01 00:00:00.0]")
+ query("select floor_datetime(date'2012-02-29', 'month')", "[2012-02-01 00:00:00.0]")
+ query("select floor_datetime(date'2012-02-29', 'week')", "[2012-02-27 00:00:00.0]")
+ query("select floor_datetime(date'2012-02-29', 'DAY')", "[2012-02-29 00:00:00.0]")
+ query("select floor_datetime(date'2012-02-29', 'hour')", "[2012-02-29 00:00:00.0]")
+ query("select floor_datetime(date'2012-02-29', 'minute')", "[2012-02-29 00:00:00.0]")
+ query("select floor_datetime(date'2012-02-29', 'second')", "[2012-02-29 00:00:00.0]")
- query("select floor_datetime('2012-02-29', 'year')").equals("[2012-01-01 00:00:00.0]")
- query("select floor_datetime('2012-02-29', 'quarter')").equals("[2012-01-01 00:00:00.0]")
- query("select floor_datetime('2012-02-29', 'month')").equals("[2012-02-01 00:00:00.0]")
- query("select floor_datetime('2012-02-29', 'week')").equals("[2012-02-27 00:00:00.0]")
- query("select floor_datetime('2012-02-29', 'DAY')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime('2012-02-29', 'hour')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime('2012-02-29', 'minute')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime('2012-02-29', 'second')").equals("[2012-02-29 00:00:00.0]")
+ query("select floor_datetime('2012-02-29', 'year')", "[2012-01-01 00:00:00.0]")
+ query("select floor_datetime('2012-02-29', 'quarter')", "[2012-01-01 00:00:00.0]")
+ query("select floor_datetime('2012-02-29', 'month')", "[2012-02-01 00:00:00.0]")
+ query("select floor_datetime('2012-02-29', 'week')", "[2012-02-27 00:00:00.0]")
+ query("select floor_datetime('2012-02-29', 'DAY')", "[2012-02-29 00:00:00.0]")
+ query("select floor_datetime('2012-02-29', 'hour')", "[2012-02-29 00:00:00.0]")
+ query("select floor_datetime('2012-02-29', 'minute')", "[2012-02-29 00:00:00.0]")
+ query("select floor_datetime('2012-02-29', 'second')", "[2012-02-29 00:00:00.0]")
- query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'year')").equals("[2012-01-01 00:00:00.0]")
- query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'quarter')").equals("[2012-01-01 00:00:00.0]")
- query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'month')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'week')").equals("[2012-02-27 00:00:00.0]")
- query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'DAY')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'hour')").equals("[2012-02-29 23:00:00.0]")
- query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'minute')").equals("[2012-02-29 23:59:00.0]")
- query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'second')").equals("[2012-02-29 23:59:59.0]")
+ query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'year')", "[2012-01-01 00:00:00.0]")
+ query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'quarter')", "[2012-01-01 00:00:00.0]")
+ query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'month')", "[2012-02-01 00:00:00.0]")
+ query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'week')", "[2012-02-27 00:00:00.0]")
+ query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'DAY')", "[2012-02-29 00:00:00.0]")
+ query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'hour')", "[2012-02-29 23:00:00.0]")
+ query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'minute')", "[2012-02-29 23:59:00.0]")
+ query("select floor_datetime(timestamp'2012-02-29 23:59:59.1', 'second')", "[2012-02-29 23:59:59.0]")
- query("select floor_datetime('2012-02-29 23:59:59.1', 'year')").equals("[2012-01-01 00:00:00.0]")
- query("select floor_datetime('2012-02-29 23:59:59.1', 'quarter')").equals("[2012-01-01 00:00:00.0]")
- query("select floor_datetime('2012-02-29 23:59:59.1', 'month')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime('2012-02-29 23:59:59.1', 'week')").equals("[2012-02-27 00:00:00.0]")
- query("select floor_datetime('2012-02-29 23:59:59.1', 'DAY')").equals("[2012-02-29 00:00:00.0]")
- query("select floor_datetime('2012-02-29 23:59:59.1', 'hour')").equals("[2012-02-29 23:00:00.0]")
- query("select floor_datetime('2012-02-29 23:59:59.1', 'minute')").equals("[2012-02-29 23:59:00.0]")
- query("select floor_datetime('2012-02-29 23:59:59.1', 'second')").equals("[2012-02-29 23:59:59.0]")
+ query("select floor_datetime('2012-02-29 23:59:59.1', 'year')", "[2012-01-01 00:00:00.0]")
+ query("select floor_datetime('2012-02-29 23:59:59.1', 'quarter')", "[2012-01-01 00:00:00.0]")
+ query("select floor_datetime('2012-02-29 23:59:59.1', 'month')", "[2012-02-01 00:00:00.0]")
+ query("select floor_datetime('2012-02-29 23:59:59.1', 'week')", "[2012-02-27 00:00:00.0]")
+ query("select floor_datetime('2012-02-29 23:59:59.1', 'DAY')", "[2012-02-29 00:00:00.0]")
+ query("select floor_datetime('2012-02-29 23:59:59.1', 'hour')", "[2012-02-29 23:00:00.0]")
+ query("select floor_datetime('2012-02-29 23:59:59.1', 'minute')", "[2012-02-29 23:59:00.0]")
+ query("select floor_datetime('2012-02-29 23:59:59.1', 'second')", "[2012-02-29 23:59:59.0]")
}
test("test spark ceil/floor") {
- query("select floor(-3.12)").equals("[3]")
+ query("select floor(-3.12)", "[-4]")
}
- def query(sql: String): String = {
- spark.sql(sql).collect().map(row => row.toString()).mkString
+ def query(sql: String, expected: String): Unit = {
+ val result = spark.sql(sql).collect().map(row => row.toString()).mkString
+ if (!result.equals(expected)) {
+ print(sql)
+ print(result)
+ }
+ assert(result.equals(expected))
}
}
diff --git a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
index bbb9d4d81c..ad72f1e6b9 100644
--- a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
+++ b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
@@ -57,16 +57,15 @@ import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MetadataChecker;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.tool.HDFSMetadataTool;
import org.apache.kylin.tool.garbage.StorageCleaner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
import lombok.var;
@@ -218,8 +217,8 @@ public class MetadataToolHelper {
}
}
- private void copyResourceStore(String projectPath, ResourceStore srcResourceStore,
- ResourceStore destResourceStore, boolean isProjectLevel, boolean excludeTableExd) {
+ private void copyResourceStore(String projectPath, ResourceStore srcResourceStore, ResourceStore destResourceStore,
+ boolean isProjectLevel, boolean excludeTableExd) {
if (excludeTableExd) {
String tableExdPath = projectPath + ResourceStore.TABLE_EXD_RESOURCE_ROOT;
var projectItems = srcResourceStore.listResources(projectPath);
@@ -373,8 +372,7 @@ public class MetadataToolHelper {
return 0;
}
- public void cleanStorage(boolean storageCleanup, List<String> projects, double requestFSRate,
- int retryTimes) {
+ public void cleanStorage(boolean storageCleanup, List<String> projects, double requestFSRate, int retryTimes) {
try {
StorageCleaner storageCleaner = new StorageCleaner(storageCleanup, projects, requestFSRate, retryTimes);
System.out.println("Start to cleanup HDFS");
@@ -394,7 +392,8 @@ public class MetadataToolHelper {
return JdbcDataSource.getDataSource(props);
}
- public void fetch(KylinConfig kylinConfig, String path, String folder, String target, boolean excludeTableExd) throws Exception {
+ public void fetch(KylinConfig kylinConfig, String path, String folder, String target, boolean excludeTableExd)
+ throws Exception {
ResourceStore resourceStore = ResourceStore.getKylinMetaStore(kylinConfig);
if (StringUtils.isBlank(path)) {
path = KylinConfigBase.getKylinHome() + File.separator + "meta_fetch";
@@ -422,7 +421,8 @@ public class MetadataToolHelper {
logger.info("start to copy target file {} from ResourceStore.", target);
UnitOfWork.doInTransactionWithRetry(
UnitOfWorkParams.builder().readonly(true).unitName(target).processor(() -> {
- copyResourceStore("/" + targetPath, resourceStore, fetchResourceStore, true, excludeTableExd);
+ copyResourceStore("/" + targetPath, resourceStore, fetchResourceStore, true,
+ excludeTableExd);
// uuid
val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG);
fetchResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(),
(kylin) 09/15: KYLIN-5741 change all epoch when empty projects request
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7cd3fd0ecc8ce95ef4f21a84abd1ed251d4e54a4
Author: Yu Gan <yu...@kyligence.io>
AuthorDate: Thu Jun 29 19:00:05 2023 +0800
KYLIN-5741 change all epoch when empty projects request
---
.../kylin/rest/controller/NEpochController.java | 9 ++++++++-
.../kylin/rest/controller/NEpochControllerTest.java | 19 ++++++++++++-------
2 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NEpochController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NEpochController.java
index 7f9f1ff00b..ac194caa73 100644
--- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NEpochController.java
+++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NEpochController.java
@@ -36,6 +36,9 @@ import org.springframework.web.bind.annotation.ResponseBody;
import io.swagger.annotations.ApiOperation;
+import java.util.ArrayList;
+import java.util.Objects;
+
@Controller
@RequestMapping(value = "/api/epoch", produces = { HTTP_VND_APACHE_KYLIN_JSON })
public class NEpochController extends NBasicController {
@@ -48,7 +51,11 @@ public class NEpochController extends NBasicController {
@PostMapping(value = "", produces = { HTTP_VND_APACHE_KYLIN_JSON, HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON })
@ResponseBody
public EnvelopeResponse<String> updateEpochOwner(@RequestBody EpochRequest epochRequest) {
- checkCollectionRequiredArg("projects", epochRequest.getProjects());
+ if (Objects.isNull(epochRequest.getProjects())) {
+ // Avoid following NPEs.
+ epochRequest.setProjects(new ArrayList<>(0));
+ }
+ // Empty projects has specified meanings: all projects do change epoch.
epochRequest.getProjects().forEach(this::checkProjectName);
checkRequiredArg("force", epochRequest.getForce());
epochService.updateEpoch(epochRequest.getProjects(), epochRequest.getForce(), epochRequest.isClient());
diff --git a/src/common-server/src/test/java/org/apache/kylin/rest/controller/NEpochControllerTest.java b/src/common-server/src/test/java/org/apache/kylin/rest/controller/NEpochControllerTest.java
index 80e0b3bf04..523d95e2c8 100644
--- a/src/common-server/src/test/java/org/apache/kylin/rest/controller/NEpochControllerTest.java
+++ b/src/common-server/src/test/java/org/apache/kylin/rest/controller/NEpochControllerTest.java
@@ -96,17 +96,12 @@ public class NEpochControllerTest extends NLocalFileMetadataTestCase {
mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
.content(JsonUtil.writeValueAsString(mapRequest2))
.contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
- .andExpect(MockMvcResultMatchers.status().is5xxServerError());
- request.setProjects(Lists.newArrayList());
- mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
- .content(JsonUtil.writeValueAsString(request))
- .contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
- .andExpect(MockMvcResultMatchers.status().is5xxServerError());
+ .andExpect(MockMvcResultMatchers.status().isOk());
request.setProjects(Lists.newArrayList());
mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
.content(JsonUtil.writeValueAsString(request))
.contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
- .andExpect(MockMvcResultMatchers.status().is5xxServerError());
+ .andExpect(MockMvcResultMatchers.status().isOk());
request.setProjects(Arrays.asList("DEFAULT"));
mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
.content(JsonUtil.writeValueAsString(request))
@@ -114,6 +109,16 @@ public class NEpochControllerTest extends NLocalFileMetadataTestCase {
.andExpect(MockMvcResultMatchers.status().isOk());
}
+ @Test
+ public void testUpdateEpochOwnerWithEmptyProjects() throws Exception {
+ val request = new EpochRequest();
+ request.setForce(false);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
+ .content(JsonUtil.writeValueAsString(request))
+ .contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ }
+
@Test
public void testUpdateAllEpochOwner() throws Exception {
val request = mockStreamingEpochRequest();
(kylin) 06/15: KYLIN-5742 Make the query result of duplicate group sets same as Spark
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c396134127b17e741e6ead1197589afe7bb773d7
Author: fengguangyuan <qq...@gmail.com>
AuthorDate: Fri Jun 30 15:28:07 2023 +0800
KYLIN-5742 Make the query result of duplicate group sets same as Spark
Co-authored-by: Guangyuan Feng <gu...@kyligence.io>
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index a61ca736d4..83adcd3e33 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,7 +135,7 @@
<scala-retry>0.3.0</scala-retry>
<!-- Calcite Version -->
- <calcite.version>1.116.0-kylin-4.x-r029</calcite.version>
+ <calcite.version>1.116.0-kylin-4.x-r031</calcite.version>
<avatica.version>4.x_1.10-r01</avatica.version>
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->