You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/02/07 07:09:42 UTC
[kylin] 13/15: [DIRTY] Revert "[DIRTY] Fix sumLC null val"
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 881e9617c155ac7b95dff2f70b7acba644b1e5d8
Author: Mingming Ge <7m...@gmail.com>
AuthorDate: Wed Nov 30 21:02:40 2022 +0800
[DIRTY] Revert "[DIRTY] Fix sumLC null val"
---
.../common/exception/code/ErrorCodeServer.java | 2 -
.../resources/kylin_error_msg_conf_cn.properties | 2 -
.../resources/kylin_error_msg_conf_en.properties | 2 -
.../kylin_error_suggestion_conf_cn.properties | 2 -
.../kylin_error_suggestion_conf_en.properties | 2 -
.../main/resources/kylin_errorcode_conf.properties | 2 -
.../apache/kylin/measure/sumlc/SumLCCounter.java | 18 --
.../apache/kylin/metadata/model/FunctionDesc.java | 33 +--
.../kylin/measure/sumlc/SumLCCounterTest.java | 54 -----
.../localmeta/data/SSB.SUMLC_CC_TEST.csv | 6 -
.../localmeta/data/SSB.SUMLC_EXTEND_4X.csv | 12 --
.../localmeta/data/SSB.SUM_LC_NULL_TBL.csv | 8 -
.../localmeta/data/SSB.SUM_LC_TB.csv | 10 -
.../4120b88e-6a3b-aba2-f86e-c692f6588f22.json | 13 --
.../f35f2937-9e4d-347a-7465-d64df939e7d6.json | 0
.../4120b88e-6a3b-aba2-f86e-c692f6588f22.json | 63 ------
.../f35f2937-9e4d-347a-7465-d64df939e7d6.json | 0
.../4120b88e-6a3b-aba2-f86e-c692f6588f22.json | 135 -------------
.../f35f2937-9e4d-347a-7465-d64df939e7d6.json | 0
.../metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json | 0
.../metadata/sum_lc/table/SSB.SUM_LC_NULL_TBL.json | 63 ------
.../org/apache/kylin/newten/SumLCResultTest.java | 195 ------------------
.../sum_lc/metadata/_global.project/sum_lc.json | 6 -
.../4120b88e-6a3b-aba2-f86e-c692f6588f22.json | 13 --
.../648098d6-3009-5b26-3e20-82e494cfdb0c.json | 17 --
.../c2f81b79-2c10-dce2-4206-588cab0e68ec.json | 17 --
.../4120b88e-6a3b-aba2-f86e-c692f6588f22.json | 63 ------
.../648098d6-3009-5b26-3e20-82e494cfdb0c.json | 63 ------
.../c2f81b79-2c10-dce2-4206-588cab0e68ec.json | 43 ----
.../4120b88e-6a3b-aba2-f86e-c692f6588f22.json | 135 -------------
.../648098d6-3009-5b26-3e20-82e494cfdb0c.json | 110 ----------
.../c2f81b79-2c10-dce2-4206-588cab0e68ec.json | 221 ---------------------
.../metadata/sum_lc/table/SSB.SUMLC_CC_TEST.json | 58 ------
.../metadata/sum_lc/table/SSB.SUM_LC_NULL_TBL.json | 63 ------
.../metadata/sum_lc/table/SSB.SUM_LC_TB.json | 53 -----
.../apache/kylin/rest/service/ModelService.java | 4 -
.../kylin/rest/service/ModelServiceTest.java | 17 --
.../kylin/query/util/DefaultQueryTransformer.java | 2 +-
.../kylin/query/engine/QueryRoutingEngine.java | 28 ++-
.../kylin/query/engine/QueryRoutingEngineTest.java | 18 +-
.../sql/catalyst/expressions/ExpressionUtils.scala | 6 +-
.../sql/catalyst/expressions/KapExpresssions.scala | 2 -
.../scala/org/apache/spark/sql/udaf/SumLC.scala | 100 ++++------
43 files changed, 66 insertions(+), 1595 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index 638b51a886..00bcf704a2 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -34,8 +34,6 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
MODEL_NOT_EXIST_SEGMENTS("KE-010002208"),
MODEL_TDS_EXPORT_DIM_COL_AND_MEASURE_NAME_CONFLICT("KE-010002301"),
MODEL_TDS_EXPORT_COLUMN_AND_MEASURE_NAME_CONFLICT("KE-010002302"),
- MODEL_SUM_LC_INVALID_DATA_TYPE("KE-010002303"),
- MODEL_SUM_LC_INVALID_TIMESTAMP_TYPE("KE-010002304"),
// 100252XX Cube
CUBE_NOT_EXIST("KE-010025201"),
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 79e5ffa746..3f1a080837 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -34,8 +34,6 @@ KE-010002207=修改模型时，simplified_measures 参数中的每个度量必
KE-010002208=模型上线必须存在 Segment。请重新输入。
KE-010002301=维度的列名 %s 与度量名 %s 重复，无法导出 TDS。请去除重名后再重试。
KE-010002302=模型中的列名 %s 与度量名 %s 重复，无法导出 TDS。请去除重名后再重试。
-KE-010002303=SUM_LC度量的返回类型 '%s' 不合法。返回类型必须是这其中的一个：%s。
-KE-010002304=SUM_LC度量的时间类型 '%s' 不合法。
## 100252XX Cube
KE-010025201=无法找到相关 Cube。
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index c44f19f10e..6f37eec3c3 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -34,8 +34,6 @@ KE-010002207=When modifying model, each measure id is required in simplified_mea
KE-010002208=The online model must have a segment. Please re-enter.
KE-010002301=There are duplicated names among dimension column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.
KE-010002302=There are duplicated names among model column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.
-KE-010002303=SUM_LC Measure's return type '%s' is illegal. It must be one of %s.
-KE-010002304=SUM_LC Measure's time column type '%s' is illegal.
## 100252XX Cube
KE-010025201=Can't find the cube.
diff --git a/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
index 6777b38ddc..10d5e8e0e4 100644
--- a/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_suggestion_conf_cn.properties
@@ -32,8 +32,6 @@ KE-010002205=
KE-010002206=
KE-010002207=
KE-010002208=
-KE-010002303=
-KE-010002304=
## 100252XX Cube
KE-010025201=
diff --git a/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties b/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
index 61b2f5c45d..1b5ffcc1c2 100644
--- a/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_suggestion_conf_en.properties
@@ -32,8 +32,6 @@ KE-010002205=
KE-010002206=
KE-010002207=
KE-010002208=
-KE-010002303=
-KE-010002304=
## 100252XX Cube
KE-010025201=
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index daad756681..ff675b94f1 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -35,8 +35,6 @@ KE-010002207
KE-010002208
KE-010002301
KE-010002302
-KE-010002303
-KE-010002304
## 100252XX Cube
KE-010025201
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCCounter.java b/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCCounter.java
index 06ed1074bd..dfca82230b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCCounter.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCCounter.java
@@ -85,24 +85,6 @@ public class SumLCCounter implements Serializable {
}
}
- public void update(Number sumLC, Long timestamp) {
- if (timestamp == null) {
- return;
- }
- Number typeConvertInput = numericTypeConversion(sumLC);
- if (this.timestamp == null || this.timestamp < timestamp) {
- this.sumLC = typeConvertInput;
- this.timestamp = timestamp;
- } else if (this.timestamp.equals(timestamp)) {
- if (this.sumLC == null) {
- this.sumLC = typeConvertInput;
- } else if (typeConvertInput != null) {
- String sumLCTypeName = this.sumLC.getClass().getSimpleName();
- this.sumLC = MERGE_FUNC_MAP.get(sumLCTypeName).apply(this.sumLC, typeConvertInput);
- }
- }
- }
-
public Number getSumLC() {
return sumLC;
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 13ec783634..28e3603d28 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -19,8 +19,6 @@
package org.apache.kylin.metadata.model;
import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_MEASURE_DATA_TYPE;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_SUM_LC_INVALID_DATA_TYPE;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_SUM_LC_INVALID_TIMESTAMP_TYPE;
import static org.apache.kylin.metadata.datatype.DataType.ANY;
import static org.apache.kylin.metadata.datatype.DataType.BIGINT;
import static org.apache.kylin.metadata.datatype.DataType.DECIMAL;
@@ -104,15 +102,16 @@ public class FunctionDesc implements Serializable {
}
break;
}
+ case FunctionDesc.FUNC_SUM_LC: {
+ Preconditions.checkArgument(StringUtils.isNotEmpty(colDataType),
+ "SUM_LC Measure's input type shouldn't be null or empty");
+ checkSumLCDataType(colDataType);
+ break;
+ }
default:
break;
}
}
- if (FunctionDesc.FUNC_SUM_LC.equals(expression)) {
- Preconditions.checkArgument(StringUtils.isNotEmpty(colDataType),
- "SUM_LC Measure's input type shouldn't be null or empty");
- checkSumLCDataType(colDataType);
- }
String returnType = override.getOrDefault(expression,
EXPRESSION_DEFAULT_TYPE_MAP.getOrDefault(expression, colDataType));
@@ -138,14 +137,9 @@ public class FunctionDesc implements Serializable {
private static void checkSumLCDataType(String dataTypeName) {
DataType dataType = DataType.getType(dataTypeName);
if (!dataType.isNumberFamily()) {
- throw new KylinException(MODEL_SUM_LC_INVALID_DATA_TYPE, dataType, DataType.NUMBER_FAMILY);
- }
- }
-
- private static void checkSumLCTimeColDataType(String dataTypeName) {
- DataType dataType = DataType.getType(dataTypeName);
- if (dataType.isTinyInt() || dataType.isFloat() || dataType.isDouble() || dataType.isDecimal() || dataType.isBoolean()) {
- throw new KylinException(MODEL_SUM_LC_INVALID_TIMESTAMP_TYPE, dataType);
+ throw new KylinException(INVALID_MEASURE_DATA_TYPE,
+ String.format(Locale.ROOT, "SUM_LC Measure's return type '%s' is illegal. It must be one of %s",
+ dataType, DataType.NUMBER_FAMILY));
}
}
@@ -216,22 +210,17 @@ public class FunctionDesc implements Serializable {
expression = PercentileMeasureType.FUNC_PERCENTILE_APPROX; // for backward compatibility
}
- List<ParameterDesc> paramList = getParameters();
- for (int i = 0; i < paramList.size(); i++) {
- ParameterDesc p = paramList.get(i);
+ for (ParameterDesc p : getParameters()) {
if (p.isColumnType()) {
TblColRef colRef = model.findColumn(p.getValue());
p.setValue(colRef.getIdentity());
p.setColRef(colRef);
if (expression.equals(FUNC_SUM_LC)) {
- if (i == 0) {
+ if (Objects.isNull(returnDataType)) {
// use the first column to init returnType and returnDataType, ignore the second timestamp column
returnType = proposeReturnType(expression, colRef.getDatatype(), Maps.newHashMap(),
model.isSaveCheck());
returnDataType = DataType.getType(returnType);
- } else {
- // check sum_lc time column type
- checkSumLCTimeColDataType(colRef.getDatatype());
}
} else {
returnDataType = DataType.getType(proposeReturnType(expression, colRef.getDatatype(),
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/measure/sumlc/SumLCCounterTest.java b/src/core-metadata/src/test/java/org/apache/kylin/measure/sumlc/SumLCCounterTest.java
deleted file mode 100644
index 800af9757e..0000000000
--- a/src/core-metadata/src/test/java/org/apache/kylin/measure/sumlc/SumLCCounterTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.measure.sumlc;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-public class SumLCCounterTest {
- private static final Long PAST_VAL = 1L;
- private static final Long PAST_TS = 915120000000L;
- private static final Long LATER_VAL = 2L;
- private static final Long LATER_TS = 1640966400000L;
-
- @Test
- public void testSumLCUpdate() {
- SumLCCounter target = new SumLCCounter();
-
- target.update(PAST_VAL, PAST_TS);
- Assert.assertEquals(PAST_VAL, target.getSumLC());
- Assert.assertEquals(PAST_TS, target.getTimestamp());
-
- target.update(null, LATER_TS);
- Assert.assertEquals(null, target.getSumLC());
- Assert.assertEquals(LATER_TS, target.getTimestamp());
-
- target.update(null, null);
- Assert.assertEquals(null, target.getSumLC());
- Assert.assertEquals(LATER_TS, target.getTimestamp());
-
- target.update(LATER_VAL, LATER_TS);
- Assert.assertEquals(LATER_VAL, target.getSumLC());
- Assert.assertEquals(LATER_TS, target.getTimestamp());
-
- target.update(PAST_VAL + LATER_VAL, LATER_TS);
- Assert.assertEquals(PAST_VAL + LATER_VAL + LATER_VAL, target.getSumLC());
- Assert.assertEquals(LATER_TS, target.getTimestamp());
- }
-
-}
diff --git a/src/examples/test_case_data/localmeta/data/SSB.SUMLC_CC_TEST.csv b/src/examples/test_case_data/localmeta/data/SSB.SUMLC_CC_TEST.csv
deleted file mode 100644
index 691965e41c..0000000000
--- a/src/examples/test_case_data/localmeta/data/SSB.SUMLC_CC_TEST.csv
+++ /dev/null
@@ -1,6 +0,0 @@
-20220901,2022,9,1,A,1
-20220902,2022,9,2,A,2
-20220903,2022,9,3,A,3
-20220901,2022,9,1,B,4
-20220902,2022,9,2,B,5
-20220903,2022,9,3,B,6
\ No newline at end of file
diff --git a/src/examples/test_case_data/localmeta/data/SSB.SUMLC_EXTEND_4X.csv b/src/examples/test_case_data/localmeta/data/SSB.SUMLC_EXTEND_4X.csv
deleted file mode 100644
index 8c6fd3ed83..0000000000
--- a/src/examples/test_case_data/localmeta/data/SSB.SUMLC_EXTEND_4X.csv
+++ /dev/null
@@ -1,12 +0,0 @@
-2022-09-01,1661961600000,B,121,32758,2147483640,2147483640,3.1415926,3.1415926,41.5343424
-2022-09-02,1662048000000,B,122,32759,2147483641,2147483641,3.1415926,3.1415926,42.5343424
-2022-09-03,1662134400000,B,123,32760,2147483642,2147483642,3.1415926,3.1415926,43.5343424
-2022-09-01,1661961600000,A,124,32761,2147483643,2147483643,3.1415926,3.1415926,44.5343424
-2022-09-02,1662048000000,A,125,32762,2147483644,2147483644,3.1415926,3.1415926,45.5343424
-2022-09-03,1662134400000,A,126,32763,2147483645,2147483645,3.1415926,3.1415926,46.5343424
-2022-10-03,1664726400000,B,121,32758,2147483640,2147483640,3.1415926,3.1415926,41.5343424
-2022-10-04,1664812800000,B,122,32759,2147483641,2147483641,3.1415926,3.1415926,42.5343424
-2022-10-05,1664899200000,B,123,32760,2147483642,2147483642,3.1415926,3.1415926,43.5343424
-2022-10-03,1664726400000,A,124,32761,2147483643,2147483643,3.1415926,3.1415926,44.5343424
-2022-10-04,1664812800000,A,125,32762,2147483644,2147483644,3.1415926,3.1415926,45.5343424
-2022-10-05,1664899200000,A,126,32763,2147483645,2147483645,3.1415926,3.1415926,46.5343424
\ No newline at end of file
diff --git a/src/examples/test_case_data/localmeta/data/SSB.SUM_LC_NULL_TBL.csv b/src/examples/test_case_data/localmeta/data/SSB.SUM_LC_NULL_TBL.csv
deleted file mode 100644
index 813b303df2..0000000000
--- a/src/examples/test_case_data/localmeta/data/SSB.SUM_LC_NULL_TBL.csv
+++ /dev/null
@@ -1,8 +0,0 @@
-A,CMB,2022-11-07,100.0,2022/11/07,100.000001,10.000001
-B,CMB,2022-11-07,110.0,2022/11/07,NULL,NULL
-C,CMB,2022-11-07,120.0,2022/11/07,NULL,NULL
-D,CMB,2022-11-07,130.0,NULL,NULL,13.000001
-A,BCM,2022-11-08,200.0,2022/11/08,200.000001,20.000001
-B,BCM,2022-11-08,210.0,2022/11/08,210.000001,21.000001
-C,BCM,2022-11-08,220.0,2022/11/08,NULL,NULL
-D,BCM,2022-11-08,230.0,2022/11/08,230.000001,23.000001
\ No newline at end of file
diff --git a/src/examples/test_case_data/localmeta/data/SSB.SUM_LC_TB.csv b/src/examples/test_case_data/localmeta/data/SSB.SUM_LC_TB.csv
deleted file mode 100644
index 78b697a9f8..0000000000
--- a/src/examples/test_case_data/localmeta/data/SSB.SUM_LC_TB.csv
+++ /dev/null
@@ -1,10 +0,0 @@
-A,CMB,2022-11-07,100,2022-11-07 00:00:00
-B,CMB,2022-11-07,110,2022-11-07 00:00:00
-A,BCM,2022-11-08,200,2022-11-08 00:00:00
-B,BCM,2022-11-08,210,2022-11-08 00:00:00
-A,ICBC,2022-11-09,300,2022-11-09 00:00:00
-B,ICBC,2022-11-09,310,2022-11-09 00:00:00
-A,ABC,2022-11-10,400,2022-11-10 00:00:00
-B,ABC,2022-11-10,410,2022-11-10 00:00:00
-A,BOC,2022-11-11,500,2022-11-11 00:00:00
-B,BOC,2022-11-11,510,2022-11-11 00:00:00
\ No newline at end of file
diff --git a/src/examples/test_case_data/localmeta/metadata/sum_lc/dataflow/4120b88e-6a3b-aba2-f86e-c692f6588f22.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/dataflow/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
deleted file mode 100644
index 8ad7bb4274..0000000000
--- a/src/examples/test_case_data/localmeta/metadata/sum_lc/dataflow/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "uuid" : "4120b88e-6a3b-aba2-f86e-c692f6588f22",
- "last_modified" : 1667836716050,
- "create_time" : 1667828178652,
- "version" : "4.0.0.0",
- "status" : "ONLINE",
- "last_status" : null,
- "cost" : 50,
- "query_hit_count" : 0,
- "last_query_time" : 0,
- "layout_query_hit_count" : { },
- "segments" : [ ]
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/f35f2937-9e4d-347a-7465-d64df939e7d6.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/dataflow/f35f2937-9e4d-347a-7465-d64df939e7d6.json
similarity index 100%
rename from src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/f35f2937-9e4d-347a-7465-d64df939e7d6.json
rename to src/examples/test_case_data/localmeta/metadata/sum_lc/dataflow/f35f2937-9e4d-347a-7465-d64df939e7d6.json
diff --git a/src/examples/test_case_data/localmeta/metadata/sum_lc/index_plan/4120b88e-6a3b-aba2-f86e-c692f6588f22.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/index_plan/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
deleted file mode 100644
index 38293ae719..0000000000
--- a/src/examples/test_case_data/localmeta/metadata/sum_lc/index_plan/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
- "uuid" : "4120b88e-6a3b-aba2-f86e-c692f6588f22",
- "last_modified" : 1668138510879,
- "create_time" : 1668138510851,
- "version" : "4.0.0.0",
- "description" : null,
- "rule_based_index" : null,
- "indexes" : [ {
- "id" : 0,
- "dimensions" : [ 0, 2, 6 ],
- "measures" : [ 100000, 100001, 100002 ],
- "layouts" : [ {
- "id" : 1,
- "name" : null,
- "owner" : null,
- "col_order" : [ 0, 2, 6, 100000, 100001, 100002 ],
- "shard_by_columns" : [ ],
- "partition_by_columns" : [ ],
- "sort_by_columns" : [ ],
- "storage_type" : 20,
- "update_time" : 1668138510858,
- "manual" : false,
- "auto" : false,
- "base" : true,
- "draft_version" : null,
- "index_range" : null
- } ],
- "next_layout_offset" : 2
- }, {
- "id" : 20000000000,
- "dimensions" : [ 0, 1, 2, 4, 5, 6 ],
- "measures" : [ ],
- "layouts" : [ {
- "id" : 20000000001,
- "name" : null,
- "owner" : null,
- "col_order" : [ 0, 1, 2, 4, 5, 6 ],
- "shard_by_columns" : [ ],
- "partition_by_columns" : [ ],
- "sort_by_columns" : [ ],
- "storage_type" : 20,
- "update_time" : 1668138510859,
- "manual" : false,
- "auto" : false,
- "base" : true,
- "draft_version" : null,
- "index_range" : null
- } ],
- "next_layout_offset" : 2
- } ],
- "override_properties" : { },
- "to_be_deleted_indexes" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 80,
- "next_aggregation_index_id" : 10000,
- "next_table_index_id" : 20000010000,
- "agg_shard_by_columns" : [ ],
- "extend_partition_columns" : [ ],
- "layout_bucket_num" : { },
- "approved_additional_recs" : 0,
- "approved_removal_recs" : 0
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/f35f2937-9e4d-347a-7465-d64df939e7d6.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/index_plan/f35f2937-9e4d-347a-7465-d64df939e7d6.json
similarity index 100%
rename from src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/f35f2937-9e4d-347a-7465-d64df939e7d6.json
rename to src/examples/test_case_data/localmeta/metadata/sum_lc/index_plan/f35f2937-9e4d-347a-7465-d64df939e7d6.json
diff --git a/src/examples/test_case_data/localmeta/metadata/sum_lc/model_desc/4120b88e-6a3b-aba2-f86e-c692f6588f22.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/model_desc/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
deleted file mode 100644
index a037ba0bd4..0000000000
--- a/src/examples/test_case_data/localmeta/metadata/sum_lc/model_desc/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
+++ /dev/null
@@ -1,135 +0,0 @@
-{
- "uuid" : "4120b88e-6a3b-aba2-f86e-c692f6588f22",
- "last_modified" : 1668138510884,
- "create_time" : 1668138510184,
- "version" : "4.0.0.0",
- "alias" : "sum_lc_null_val_test",
- "owner" : "ADMIN",
- "config_last_modifier" : null,
- "config_last_modified" : 0,
- "description" : null,
- "fact_table" : "SSB.SUM_LC_NULL_TBL",
- "fact_table_alias" : null,
- "management_type" : "MODEL_BASED",
- "join_tables" : [ ],
- "filter_condition" : "",
- "partition_desc" : {
- "partition_date_column" : "SUM_LC_NULL_TBL.PART_COL",
- "partition_date_start" : 0,
- "partition_date_format" : "yyyy-MM-dd",
- "partition_type" : "APPEND",
- "partition_condition_builder" : "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
- },
- "capacity" : "MEDIUM",
- "segment_config" : {
- "auto_merge_enabled" : null,
- "auto_merge_time_ranges" : null,
- "volatile_range" : null,
- "retention_range" : null,
- "create_empty_segment_enabled" : false
- },
- "data_check_desc" : null,
- "semantic_version" : 0,
- "storage_type" : 0,
- "model_type" : "BATCH",
- "all_named_columns" : [ {
- "id" : 0,
- "name" : "PART_COL",
- "column" : "SUM_LC_NULL_TBL.PART_COL",
- "status" : "DIMENSION"
- }, {
- "id" : 1,
- "name" : "SUM_DATE1",
- "column" : "SUM_LC_NULL_TBL.SUM_DATE1"
- }, {
- "id" : 2,
- "name" : "ACCOUNT1",
- "column" : "SUM_LC_NULL_TBL.ACCOUNT1",
- "status" : "DIMENSION"
- }, {
- "id" : 3,
- "name" : "BALANCE1",
- "column" : "SUM_LC_NULL_TBL.BALANCE1"
- }, {
- "id" : 4,
- "name" : "DATA_NULL",
- "column" : "SUM_LC_NULL_TBL.DATA_NULL"
- }, {
- "id" : 5,
- "name" : "DATA_DECIMAL",
- "column" : "SUM_LC_NULL_TBL.DATA_DECIMAL"
- }, {
- "id" : 6,
- "name" : "ACCOUNT2",
- "column" : "SUM_LC_NULL_TBL.ACCOUNT2",
- "status" : "DIMENSION"
- } ],
- "all_measures" : [ {
- "name" : "COUNT_ALL",
- "function" : {
- "expression" : "COUNT",
- "parameters" : [ {
- "type" : "constant",
- "value" : "1"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : null,
- "id" : 100000,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sumlc_double_null",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUM_LC_NULL_TBL.DATA_NULL"
- }, {
- "type" : "column",
- "value" : "SUM_LC_NULL_TBL.SUM_DATE1"
- } ],
- "returntype" : "double"
- },
- "column" : null,
- "comment" : "",
- "id" : 100001,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sumlc_decimal_null",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUM_LC_NULL_TBL.DATA_DECIMAL"
- }, {
- "type" : "column",
- "value" : "SUM_LC_NULL_TBL.SUM_DATE1"
- } ],
- "returntype" : "decimal(20,6)"
- },
- "column" : null,
- "comment" : "",
- "id" : 100002,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- } ],
- "recommendations_count" : 0,
- "computed_columns" : [ ],
- "canvas" : {
- "coordinate" : {
- "SUM_LC_NULL_TBL" : {
- "x" : 462.44444105360253,
- "y" : 108.66667005750864,
- "width" : 200.0,
- "height" : 230.0
- }
- },
- "zoom" : 9.0
- },
- "multi_partition_desc" : null,
- "multi_partition_key_mapping" : null,
- "fusion_id" : null
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/f35f2937-9e4d-347a-7465-d64df939e7d6.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/model_desc/f35f2937-9e4d-347a-7465-d64df939e7d6.json
similarity index 100%
rename from src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/f35f2937-9e4d-347a-7465-d64df939e7d6.json
rename to src/examples/test_case_data/localmeta/metadata/sum_lc/model_desc/f35f2937-9e4d-347a-7465-d64df939e7d6.json
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json
similarity index 100%
rename from src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json
rename to src/examples/test_case_data/localmeta/metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json
diff --git a/src/examples/test_case_data/localmeta/metadata/sum_lc/table/SSB.SUM_LC_NULL_TBL.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/table/SSB.SUM_LC_NULL_TBL.json
deleted file mode 100644
index be31cb96db..0000000000
--- a/src/examples/test_case_data/localmeta/metadata/sum_lc/table/SSB.SUM_LC_NULL_TBL.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
- "uuid" : "6d164a7b-de25-e326-32f3-92103e53cf7e",
- "last_modified" : 0,
- "create_time" : 1668136653291,
- "version" : "4.0.0.0",
- "name" : "SUM_LC_NULL_TBL",
- "columns" : [ {
- "id" : "1",
- "name" : "ACCOUNT1",
- "datatype" : "char(1)",
- "case_sensitive_name" : "account1"
- }, {
- "id" : "2",
- "name" : "ACCOUNT2",
- "datatype" : "varchar(52)",
- "case_sensitive_name" : "account2"
- }, {
- "id" : "3",
- "name" : "PART_COL",
- "datatype" : "date",
- "case_sensitive_name" : "part_col"
- }, {
- "id" : "4",
- "name" : "BALANCE1",
- "datatype" : "double",
- "case_sensitive_name" : "balance1"
- }, {
- "id" : "5",
- "name" : "SUM_DATE1",
- "datatype" : "varchar(1024)",
- "case_sensitive_name" : "sum_date1"
- }, {
- "id" : "6",
- "name" : "DATA_NULL",
- "datatype" : "double",
- "case_sensitive_name" : "data_null"
- }, {
- "id" : "7",
- "name" : "DATA_DECIMAL",
- "datatype" : "decimal(10,6)",
- "case_sensitive_name" : "data_decimal"
- } ],
- "source_type" : 9,
- "table_type" : "MANAGED",
- "top" : false,
- "increment_loading" : false,
- "last_snapshot_path" : null,
- "last_snapshot_size" : 0,
- "snapshot_last_modified" : 0,
- "query_hit_count" : 0,
- "partition_column" : null,
- "snapshot_partitions" : { },
- "snapshot_partitions_info" : { },
- "snapshot_total_rows" : 0,
- "snapshot_partition_col" : null,
- "selected_snapshot_partition_col" : null,
- "temp_snapshot_path" : null,
- "snapshot_has_broken" : false,
- "database" : "SSB",
- "transactional" : false,
- "rangePartition" : false,
- "partition_desc" : null
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/SumLCResultTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/SumLCResultTest.java
deleted file mode 100644
index 4d2bd40fa3..0000000000
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/SumLCResultTest.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.newten;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
-import org.apache.kylin.metadata.cube.model.NDataflow;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
-import org.apache.kylin.metadata.model.SegmentRange;
-import org.apache.kylin.util.ExecAndComp;
-import org.apache.spark.sql.SparderEnv;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Sets;
-
-public class SumLCResultTest extends NLocalWithSparkSessionTest {
-
- private NDataflowManager dfMgr = null;
-
- @Before
- public void setup() throws Exception {
- overwriteSystemProp("kylin.job.scheduler.poll-interval-second", "1");
- this.createTestMetadata("src/test/resources/ut_meta/sum_lc");
- dfMgr = NDataflowManager.getInstance(getTestConfig(), getProject());
- NDefaultScheduler scheduler = NDefaultScheduler.getInstance(getProject());
- scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- }
-
- @After
- public void after() throws Exception {
- NDefaultScheduler.destroyInstance();
- cleanupTestMetadata();
- }
-
- @Override
- public String getProject() {
- return "sum_lc";
- }
-
- @Test
- public void testSumLCWithDifferentDataType() throws Exception {
- String dfID = "f35f2937-9e4d-347a-7465-d64df939e7d6";
- NDataflow dataflow = dfMgr.getDataflow(dfID);
- indexDataConstructor.buildIndex(dfID, new SegmentRange.TimePartitionedSegmentRange(1661961600000L, 1664553600000L),
- Sets.newHashSet(dataflow.getIndexPlan().getLayoutEntity(30001L)), true);
-
- populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession());
- List<Pair<String, String>> query = new ArrayList<>();
- String sql1 = "select TX_DATE, sum_lc(TINYINT_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X group by TX_DATE";
- query.add(Pair.newPair("sum_lc_tinyint_data_query", sql1));
-
- String sql2 = "select TX_DATE, sum_lc(SMALLINT_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X group by TX_DATE";
- query.add(Pair.newPair("sum_lc_smallint_data_query", sql2));
-
- String sql3 = "select TX_DATE, sum_lc(INT_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X group by TX_DATE";
- query.add(Pair.newPair("sum_lc_int_data_query", sql3));
-
- String sql4 = "select TX_DATE, sum_lc(BIGINT_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X group by TX_DATE";
- query.add(Pair.newPair("sum_lc_bigint_data_query", sql4));
-
- String sql5 = "select TX_DATE, sum_lc(FLOAT_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X group by TX_DATE";
- query.add(Pair.newPair("sum_lc_float_data_query", sql5));
-
- String sql6 = "select TX_DATE, sum_lc(DOUBLE_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X group by TX_DATE";
- query.add(Pair.newPair("sum_lc_double_data_query", sql6));
-
- String sql7 = "select TX_DATE, sum_lc(DECIMAL_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X group by TX_DATE";
- query.add(Pair.newPair("sum_lc_decimal_data_query", sql7));
-
- ExecAndComp.execAndCompare(query, getProject(), ExecAndComp.CompareLevel.NONE, "left");
- }
-
- @Test
- public void testPostAggregate() throws Exception {
- String dfID = "f35f2937-9e4d-347a-7465-d64df939e7d6";
- NDataflow dataflow = dfMgr.getDataflow(dfID);
- indexDataConstructor.buildIndex(dfID, new SegmentRange.TimePartitionedSegmentRange(1661961600000L, 1664553600000L),
- Sets.newHashSet(dataflow.getIndexPlan().getLayoutEntity(30001L)), true);
-
- populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession());
- List<Pair<String, String>> query = new ArrayList<>();
-
- String sql = "select sum_lc(TINYINT_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X";
- query.add(Pair.newPair("sum_lc_post_aggregate_query", sql));
-
- ExecAndComp.execAndCompare(query, getProject(), ExecAndComp.CompareLevel.NONE, "left");
- }
-
- @Test
- public void testEmptySegment() throws Exception {
- String dfID = "f35f2937-9e4d-347a-7465-d64df939e7d6";
- NDataflow dataflow = dfMgr.getDataflow(dfID);
- indexDataConstructor.buildIndex(dfID, new SegmentRange.TimePartitionedSegmentRange(1667232000000L, 1669824000000L),
- Sets.newHashSet(dataflow.getIndexPlan().getLayoutEntity(50001L)), true);
-
- populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession());
- List<Pair<String, String>> query = new ArrayList<>();
-
- String sql1 = "select account, sum_lc(TINYINT_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X group by account";
- query.add(Pair.newPair("sum_lc_empty_seg_exactly_match_query", sql1));
-
- String sql2 = "select sum_lc(TINYINT_DATA, TX_DATE) from SSB.SUMLC_EXTEND_4X";
- query.add(Pair.newPair("sum_lc_empty_seg_post_agg_query", sql2));
-
- ExecAndComp.execAndCompare(query, getProject(), ExecAndComp.CompareLevel.NONE, "left");
- }
-
- @Test
- public void testCCQuery() throws Exception {
- String dfID = "c2f81b79-2c10-dce2-4206-588cab0e68ec";
- NDataflow dataflow = dfMgr.getDataflow(dfID);
- indexDataConstructor.buildIndex(dfID, SegmentRange.TimePartitionedSegmentRange.createInfinite(),
- Sets.newHashSet(dataflow.getIndexPlan().getLayoutEntity(190001L)), true);
-
- populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession());
- List<Pair<String, String>> query = new ArrayList<>();
-
- String sql1 = "select account, sum_lc(INT_DATA, TO_TIME_COMPOSE_CC) from SSB.SUMLC_CC_TEST group by account";
- query.add(Pair.newPair("sum_lc_cc_time", sql1));
-
- String sql2 = "select account, sum_lc(INT_DATA * 2, TO_TIME_COMPOSE_CC) from SSB.SUMLC_CC_TEST group by account";
- query.add(Pair.newPair("sum_lc_cc_int", sql2));
-
- String sql3 = "select account, sum_lc(INT_DATA_CC, TO_TIME_COMPOSE_CC) from SSB.SUMLC_CC_TEST group by account";
- query.add(Pair.newPair("sum_lc_cc_int_cc_time", sql3));
-
- ExecAndComp.execAndCompare(query, getProject(), ExecAndComp.CompareLevel.NONE, "left");
- }
-
- @Test
- public void testTimestampTimeCol() throws Exception {
- String dfID = "648098d6-3009-5b26-3e20-82e494cfdb0c";
- NDataflow dataflow = dfMgr.getDataflow(dfID);
- indexDataConstructor.buildIndex(dfID, SegmentRange.TimePartitionedSegmentRange.createInfinite(),
- Sets.newHashSet(dataflow.getIndexPlan().getLayoutEntity(1L)), true);
-
- populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession());
- List<Pair<String, String>> query = new ArrayList<>();
-
- String sql = "select sum_lc(BALANCE, SUM_DATE) from SSB.SUM_LC_TB";
- query.add(Pair.newPair("query", sql));
- ExecAndComp.execAndCompare(query, getProject(), ExecAndComp.CompareLevel.NONE, "left");
- }
-
- @Test
- public void testNullVal() throws Exception {
- String dfID = "4120b88e-6a3b-aba2-f86e-c692f6588f22";
- NDataflow df = dfMgr.getDataflow(dfID);
- indexDataConstructor.buildIndex(dfID,
- new SegmentRange.TimePartitionedSegmentRange(1667750400000L, 1667836800000L),
- Sets.newHashSet(df.getIndexPlan().getLayoutEntity(1L)), true);
-
- populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession());
- List<Pair<String, String>> query = new ArrayList<>();
-
- String sql1 = "select account1, account2, part_col, sum_lc(data_null, sum_date1) from ssb.sum_lc_null_tbl group by account1, account2, part_col";
- query.add(Pair.newPair("exact_match_null_query", sql1));
-
- String sql2 = "select account1, sum_lc(data_null, sum_date1) from ssb.sum_lc_null_tbl group by account1";
- query.add(Pair.newPair("double_null_query", sql2));
-
- String sql3 = "select account1, sum_lc(data_decimal, sum_date1) from ssb.sum_lc_null_tbl group by account1";
- query.add(Pair.newPair("decimal_null_query", sql3));
-
- ExecAndComp.execAndCompare(query, getProject(), ExecAndComp.CompareLevel.NONE, "left");
- }
-
-}
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/_global.project/sum_lc.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/_global.project/sum_lc.json
deleted file mode 100644
index 339191f0c8..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/_global.project/sum_lc.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "uuid": "d1ceb3a4-4d2c-27af-5b7d-92c8a4055776",
- "override_kylin_properties": {
- "kylin.query.slowquery-detect-interval": "4"
- }
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/4120b88e-6a3b-aba2-f86e-c692f6588f22.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
deleted file mode 100644
index 8ad7bb4274..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "uuid" : "4120b88e-6a3b-aba2-f86e-c692f6588f22",
- "last_modified" : 1667836716050,
- "create_time" : 1667828178652,
- "version" : "4.0.0.0",
- "status" : "ONLINE",
- "last_status" : null,
- "cost" : 50,
- "query_hit_count" : 0,
- "last_query_time" : 0,
- "layout_query_hit_count" : { },
- "segments" : [ ]
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/648098d6-3009-5b26-3e20-82e494cfdb0c.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/648098d6-3009-5b26-3e20-82e494cfdb0c.json
deleted file mode 100644
index e8dce17945..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/648098d6-3009-5b26-3e20-82e494cfdb0c.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "uuid" : "648098d6-3009-5b26-3e20-82e494cfdb0c",
- "last_modified" : 1667836716050,
- "create_time" : 1667828178652,
- "version" : "4.0.0.0",
- "status" : "ONLINE",
- "last_status" : null,
- "cost" : 50,
- "query_hit_count" : 6,
- "last_query_time" : 1667835303109,
- "layout_query_hit_count" : {
- "1" : {
- "1667750400000" : 6
- }
- },
- "segments" : [ ]
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/c2f81b79-2c10-dce2-4206-588cab0e68ec.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/c2f81b79-2c10-dce2-4206-588cab0e68ec.json
deleted file mode 100644
index 2966d75878..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/dataflow/c2f81b79-2c10-dce2-4206-588cab0e68ec.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "uuid" : "c2f81b79-2c10-dce2-4206-588cab0e68ec",
- "last_modified" : 1667549287312,
- "create_time" : 1667544082554,
- "version" : "4.0.0.0",
- "status" : "ONLINE",
- "last_status" : null,
- "cost" : 50,
- "query_hit_count" : 3,
- "last_query_time" : 1667549256856,
- "layout_query_hit_count" : {
- "190001" : {
- "1667491200000" : 3
- }
- },
- "segments" : [ ]
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/4120b88e-6a3b-aba2-f86e-c692f6588f22.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
deleted file mode 100644
index 38293ae719..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
- "uuid" : "4120b88e-6a3b-aba2-f86e-c692f6588f22",
- "last_modified" : 1668138510879,
- "create_time" : 1668138510851,
- "version" : "4.0.0.0",
- "description" : null,
- "rule_based_index" : null,
- "indexes" : [ {
- "id" : 0,
- "dimensions" : [ 0, 2, 6 ],
- "measures" : [ 100000, 100001, 100002 ],
- "layouts" : [ {
- "id" : 1,
- "name" : null,
- "owner" : null,
- "col_order" : [ 0, 2, 6, 100000, 100001, 100002 ],
- "shard_by_columns" : [ ],
- "partition_by_columns" : [ ],
- "sort_by_columns" : [ ],
- "storage_type" : 20,
- "update_time" : 1668138510858,
- "manual" : false,
- "auto" : false,
- "base" : true,
- "draft_version" : null,
- "index_range" : null
- } ],
- "next_layout_offset" : 2
- }, {
- "id" : 20000000000,
- "dimensions" : [ 0, 1, 2, 4, 5, 6 ],
- "measures" : [ ],
- "layouts" : [ {
- "id" : 20000000001,
- "name" : null,
- "owner" : null,
- "col_order" : [ 0, 1, 2, 4, 5, 6 ],
- "shard_by_columns" : [ ],
- "partition_by_columns" : [ ],
- "sort_by_columns" : [ ],
- "storage_type" : 20,
- "update_time" : 1668138510859,
- "manual" : false,
- "auto" : false,
- "base" : true,
- "draft_version" : null,
- "index_range" : null
- } ],
- "next_layout_offset" : 2
- } ],
- "override_properties" : { },
- "to_be_deleted_indexes" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 80,
- "next_aggregation_index_id" : 10000,
- "next_table_index_id" : 20000010000,
- "agg_shard_by_columns" : [ ],
- "extend_partition_columns" : [ ],
- "layout_bucket_num" : { },
- "approved_additional_recs" : 0,
- "approved_removal_recs" : 0
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/648098d6-3009-5b26-3e20-82e494cfdb0c.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/648098d6-3009-5b26-3e20-82e494cfdb0c.json
deleted file mode 100644
index b28d6b4543..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/648098d6-3009-5b26-3e20-82e494cfdb0c.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
- "uuid" : "648098d6-3009-5b26-3e20-82e494cfdb0c",
- "last_modified" : 1667835213989,
- "create_time" : 1667828178631,
- "version" : "4.0.0.0",
- "description" : null,
- "rule_based_index" : null,
- "indexes" : [ {
- "id" : 0,
- "dimensions" : [ 0, 1, 2, 4 ],
- "measures" : [ 100000, 100001 ],
- "layouts" : [ {
- "id" : 1,
- "name" : null,
- "owner" : null,
- "col_order" : [ 0, 1, 2, 4, 100000, 100001 ],
- "shard_by_columns" : [ ],
- "partition_by_columns" : [ ],
- "sort_by_columns" : [ ],
- "storage_type" : 20,
- "update_time" : 1667828178631,
- "manual" : false,
- "auto" : false,
- "base" : true,
- "draft_version" : null,
- "index_range" : null
- } ],
- "next_layout_offset" : 2
- }, {
- "id" : 20000000000,
- "dimensions" : [ 0, 1, 2, 3, 4 ],
- "measures" : [ ],
- "layouts" : [ {
- "id" : 20000000001,
- "name" : null,
- "owner" : null,
- "col_order" : [ 0, 1, 2, 3, 4 ],
- "shard_by_columns" : [ ],
- "partition_by_columns" : [ ],
- "sort_by_columns" : [ ],
- "storage_type" : 20,
- "update_time" : 1667828178632,
- "manual" : false,
- "auto" : false,
- "base" : true,
- "draft_version" : null,
- "index_range" : null
- } ],
- "next_layout_offset" : 2
- } ],
- "override_properties" : { },
- "to_be_deleted_indexes" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 80,
- "next_aggregation_index_id" : 10000,
- "next_table_index_id" : 20000010000,
- "agg_shard_by_columns" : [ ],
- "extend_partition_columns" : [ ],
- "layout_bucket_num" : { },
- "approved_additional_recs" : 0,
- "approved_removal_recs" : 0
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/c2f81b79-2c10-dce2-4206-588cab0e68ec.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/c2f81b79-2c10-dce2-4206-588cab0e68ec.json
deleted file mode 100644
index af9ed2c9d5..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/index_plan/c2f81b79-2c10-dce2-4206-588cab0e68ec.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid" : "c2f81b79-2c10-dce2-4206-588cab0e68ec",
- "last_modified" : 1667548832263,
- "create_time" : 1667544082554,
- "version" : "4.0.0.0",
- "description" : null,
- "rule_based_index" : {
- "dimensions" : [ 0, 5, 7 ],
- "measures" : [ 100000, 100002, 100004, 100005 ],
- "global_dim_cap" : null,
- "aggregation_groups" : [ {
- "includes" : [ 0, 5, 7 ],
- "measures" : [ 100000, 100002, 100004, 100005 ],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ ],
- "joint_dims" : [ ]
- },
- "index_range" : "EMPTY"
- } ],
- "layout_id_mapping" : [ 150001, 160001, 170001, 180001, 190001, 200001, 210001 ],
- "parent_forward" : 3,
- "index_start_id" : 150000,
- "last_modify_time" : 1667548832258,
- "layout_black_list" : [ ],
- "scheduler_version" : 2,
- "index_update_enabled" : true,
- "base_layout_enabled" : true
- },
- "indexes" : [ ],
- "override_properties" : { },
- "to_be_deleted_indexes" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 80,
- "next_aggregation_index_id" : 220000,
- "next_table_index_id" : 20000000000,
- "agg_shard_by_columns" : [ ],
- "extend_partition_columns" : [ ],
- "layout_bucket_num" : { },
- "approved_additional_recs" : 0,
- "approved_removal_recs" : 0
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/4120b88e-6a3b-aba2-f86e-c692f6588f22.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
deleted file mode 100644
index a037ba0bd4..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/4120b88e-6a3b-aba2-f86e-c692f6588f22.json
+++ /dev/null
@@ -1,135 +0,0 @@
-{
- "uuid" : "4120b88e-6a3b-aba2-f86e-c692f6588f22",
- "last_modified" : 1668138510884,
- "create_time" : 1668138510184,
- "version" : "4.0.0.0",
- "alias" : "sum_lc_null_val_test",
- "owner" : "ADMIN",
- "config_last_modifier" : null,
- "config_last_modified" : 0,
- "description" : null,
- "fact_table" : "SSB.SUM_LC_NULL_TBL",
- "fact_table_alias" : null,
- "management_type" : "MODEL_BASED",
- "join_tables" : [ ],
- "filter_condition" : "",
- "partition_desc" : {
- "partition_date_column" : "SUM_LC_NULL_TBL.PART_COL",
- "partition_date_start" : 0,
- "partition_date_format" : "yyyy-MM-dd",
- "partition_type" : "APPEND",
- "partition_condition_builder" : "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
- },
- "capacity" : "MEDIUM",
- "segment_config" : {
- "auto_merge_enabled" : null,
- "auto_merge_time_ranges" : null,
- "volatile_range" : null,
- "retention_range" : null,
- "create_empty_segment_enabled" : false
- },
- "data_check_desc" : null,
- "semantic_version" : 0,
- "storage_type" : 0,
- "model_type" : "BATCH",
- "all_named_columns" : [ {
- "id" : 0,
- "name" : "PART_COL",
- "column" : "SUM_LC_NULL_TBL.PART_COL",
- "status" : "DIMENSION"
- }, {
- "id" : 1,
- "name" : "SUM_DATE1",
- "column" : "SUM_LC_NULL_TBL.SUM_DATE1"
- }, {
- "id" : 2,
- "name" : "ACCOUNT1",
- "column" : "SUM_LC_NULL_TBL.ACCOUNT1",
- "status" : "DIMENSION"
- }, {
- "id" : 3,
- "name" : "BALANCE1",
- "column" : "SUM_LC_NULL_TBL.BALANCE1"
- }, {
- "id" : 4,
- "name" : "DATA_NULL",
- "column" : "SUM_LC_NULL_TBL.DATA_NULL"
- }, {
- "id" : 5,
- "name" : "DATA_DECIMAL",
- "column" : "SUM_LC_NULL_TBL.DATA_DECIMAL"
- }, {
- "id" : 6,
- "name" : "ACCOUNT2",
- "column" : "SUM_LC_NULL_TBL.ACCOUNT2",
- "status" : "DIMENSION"
- } ],
- "all_measures" : [ {
- "name" : "COUNT_ALL",
- "function" : {
- "expression" : "COUNT",
- "parameters" : [ {
- "type" : "constant",
- "value" : "1"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : null,
- "id" : 100000,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sumlc_double_null",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUM_LC_NULL_TBL.DATA_NULL"
- }, {
- "type" : "column",
- "value" : "SUM_LC_NULL_TBL.SUM_DATE1"
- } ],
- "returntype" : "double"
- },
- "column" : null,
- "comment" : "",
- "id" : 100001,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sumlc_decimal_null",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUM_LC_NULL_TBL.DATA_DECIMAL"
- }, {
- "type" : "column",
- "value" : "SUM_LC_NULL_TBL.SUM_DATE1"
- } ],
- "returntype" : "decimal(20,6)"
- },
- "column" : null,
- "comment" : "",
- "id" : 100002,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- } ],
- "recommendations_count" : 0,
- "computed_columns" : [ ],
- "canvas" : {
- "coordinate" : {
- "SUM_LC_NULL_TBL" : {
- "x" : 462.44444105360253,
- "y" : 108.66667005750864,
- "width" : 200.0,
- "height" : 230.0
- }
- },
- "zoom" : 9.0
- },
- "multi_partition_desc" : null,
- "multi_partition_key_mapping" : null,
- "fusion_id" : null
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/648098d6-3009-5b26-3e20-82e494cfdb0c.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/648098d6-3009-5b26-3e20-82e494cfdb0c.json
deleted file mode 100644
index 660c1420fe..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/648098d6-3009-5b26-3e20-82e494cfdb0c.json
+++ /dev/null
@@ -1,110 +0,0 @@
-{
- "uuid" : "648098d6-3009-5b26-3e20-82e494cfdb0c",
- "last_modified" : 1667828178657,
- "create_time" : 1667828178249,
- "version" : "4.0.0.0",
- "alias" : "sum_lc_timestamp_test",
- "owner" : "ADMIN",
- "config_last_modifier" : null,
- "config_last_modified" : 0,
- "description" : null,
- "fact_table" : "SSB.SUM_LC_TB",
- "fact_table_alias" : null,
- "management_type" : "MODEL_BASED",
- "join_tables" : [ ],
- "filter_condition" : "",
- "partition_desc" : {
- "partition_date_column" : "SUM_LC_TB.PART_COL",
- "partition_date_start" : 0,
- "partition_date_format" : "yyyy-MM-dd",
- "partition_type" : "APPEND",
- "partition_condition_builder" : "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
- },
- "capacity" : "MEDIUM",
- "segment_config" : {
- "auto_merge_enabled" : null,
- "auto_merge_time_ranges" : null,
- "volatile_range" : null,
- "retention_range" : null,
- "create_empty_segment_enabled" : false
- },
- "data_check_desc" : null,
- "semantic_version" : 0,
- "storage_type" : 0,
- "model_type" : "BATCH",
- "all_named_columns" : [ {
- "id" : 0,
- "name" : "ACCOUNT",
- "column" : "SUM_LC_TB.ACCOUNT",
- "status" : "DIMENSION"
- }, {
- "id" : 1,
- "name" : "BANK_NAME",
- "column" : "SUM_LC_TB.BANK_NAME",
- "status" : "DIMENSION"
- }, {
- "id" : 2,
- "name" : "PART_COL",
- "column" : "SUM_LC_TB.PART_COL",
- "status" : "DIMENSION"
- }, {
- "id" : 3,
- "name" : "BALANCE",
- "column" : "SUM_LC_TB.BALANCE"
- }, {
- "id" : 4,
- "name" : "SUM_DATE",
- "column" : "SUM_LC_TB.SUM_DATE",
- "status" : "DIMENSION"
- } ],
- "all_measures" : [ {
- "name" : "COUNT_ALL",
- "function" : {
- "expression" : "COUNT",
- "parameters" : [ {
- "type" : "constant",
- "value" : "1"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : null,
- "id" : 100000,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sum_lc_balance",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUM_LC_TB.BALANCE"
- }, {
- "type" : "column",
- "value" : "SUM_LC_TB.SUM_DATE"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : "",
- "id" : 100001,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- } ],
- "recommendations_count" : 0,
- "computed_columns" : [ ],
- "canvas" : {
- "coordinate" : {
- "SUM_LC_TB" : {
- "x" : 585.7777913411459,
- "y" : 87.44444105360242,
- "width" : 200.0,
- "height" : 483.33333333333337
- }
- },
- "zoom" : 9.0
- },
- "multi_partition_desc" : null,
- "multi_partition_key_mapping" : null,
- "fusion_id" : null
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/c2f81b79-2c10-dce2-4206-588cab0e68ec.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/c2f81b79-2c10-dce2-4206-588cab0e68ec.json
deleted file mode 100644
index 000b6bcce7..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/model_desc/c2f81b79-2c10-dce2-4206-588cab0e68ec.json
+++ /dev/null
@@ -1,221 +0,0 @@
-{
- "uuid" : "c2f81b79-2c10-dce2-4206-588cab0e68ec",
- "last_modified" : 1667548802841,
- "create_time" : 1667544081868,
- "version" : "4.0.0.0",
- "alias" : "sum_lc_cc_test",
- "owner" : "ADMIN",
- "config_last_modifier" : null,
- "config_last_modified" : 0,
- "description" : null,
- "fact_table" : "SSB.SUMLC_CC_TEST",
- "fact_table_alias" : null,
- "management_type" : "MODEL_BASED",
- "join_tables" : [ ],
- "filter_condition" : "",
- "partition_desc" : null,
- "capacity" : "MEDIUM",
- "segment_config" : {
- "auto_merge_enabled" : null,
- "auto_merge_time_ranges" : null,
- "volatile_range" : null,
- "retention_range" : null,
- "create_empty_segment_enabled" : false
- },
- "data_check_desc" : null,
- "semantic_version" : 0,
- "storage_type" : 0,
- "model_type" : "BATCH",
- "all_named_columns" : [ {
- "id" : 0,
- "name" : "ACCOUNT",
- "column" : "SUMLC_CC_TEST.ACCOUNT",
- "status" : "DIMENSION"
- }, {
- "id" : 1,
- "name" : "MONTH_DATA",
- "column" : "SUMLC_CC_TEST.MONTH_DATA"
- }, {
- "id" : 2,
- "name" : "INT_DATA",
- "column" : "SUMLC_CC_TEST.INT_DATA"
- }, {
- "id" : 3,
- "name" : "YEAR_DATA",
- "column" : "SUMLC_CC_TEST.YEAR_DATA"
- }, {
- "id" : 4,
- "name" : "DAY_DATA",
- "column" : "SUMLC_CC_TEST.DAY_DATA"
- }, {
- "id" : 5,
- "name" : "DATE_DATA",
- "column" : "SUMLC_CC_TEST.DATE_DATA",
- "status" : "DIMENSION"
- }, {
- "id" : 6,
- "name" : "TIME_COMPOSE_CC",
- "column" : "SUMLC_CC_TEST.TIME_COMPOSE_CC"
- }, {
- "id" : 7,
- "name" : "INT_DATA_CC",
- "column" : "SUMLC_CC_TEST.INT_DATA_CC",
- "status" : "DIMENSION"
- }, {
- "id" : 8,
- "name" : "TO_TIME_COMPOSE_CC",
- "column" : "SUMLC_CC_TEST.TO_TIME_COMPOSE_CC",
- "status" : "DIMENSION"
- } ],
- "all_measures" : [ {
- "name" : "COUNT_ALL",
- "function" : {
- "expression" : "COUNT",
- "parameters" : [ {
- "type" : "constant",
- "value" : "1"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : null,
- "id" : 100000,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sum_lc_int_data_time_cc",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.INT_DATA"
- }, {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.TIME_COMPOSE_CC"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : "",
- "id" : 100001,
- "tomb" : true,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sum_lc_int_data_cc_date_data",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.INT_DATA_CC"
- }, {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.DATE_DATA"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : "",
- "id" : 100002,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sum_lc_int_cc_time_cc",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.INT_DATA_CC"
- }, {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.TIME_COMPOSE_CC"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : "",
- "id" : 100003,
- "tomb" : true,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sum_lc_int_data_time_cc",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.INT_DATA"
- }, {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.TO_TIME_COMPOSE_CC"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : "",
- "id" : 100004,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- }, {
- "name" : "sum_lc_int_cc_time_cc",
- "function" : {
- "expression" : "SUM_LC",
- "parameters" : [ {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.INT_DATA_CC"
- }, {
- "type" : "column",
- "value" : "SUMLC_CC_TEST.TO_TIME_COMPOSE_CC"
- } ],
- "returntype" : "bigint"
- },
- "column" : null,
- "comment" : "",
- "id" : 100005,
- "type" : "NORMAL",
- "internal_ids" : [ ]
- } ],
- "recommendations_count" : 0,
- "computed_columns" : [ {
- "tableIdentity" : "SSB.SUMLC_CC_TEST",
- "tableAlias" : "SUMLC_CC_TEST",
- "columnName" : "TIME_COMPOSE_CC",
- "expression" : "concat(SUMLC_CC_TEST.YEAR_DATA,'-',SUMLC_CC_TEST.MONTH_DATA,'-',SUMLC_CC_TEST.DAY_DATA)",
- "innerExpression" : "concat(`SUMLC_CC_TEST`.`YEAR_DATA`, '-', `SUMLC_CC_TEST`.`MONTH_DATA`, '-', `SUMLC_CC_TEST`.`DAY_DATA`)",
- "datatype" : "VARCHAR",
- "comment" : null,
- "rec_uuid" : null
- }, {
- "tableIdentity" : "SSB.SUMLC_CC_TEST",
- "tableAlias" : "SUMLC_CC_TEST",
- "columnName" : "INT_DATA_CC",
- "expression" : "SUMLC_CC_TEST.INT_DATA * 2",
- "innerExpression" : "`SUMLC_CC_TEST`.`INT_DATA` * 2",
- "datatype" : "INTEGER",
- "comment" : null,
- "rec_uuid" : null
- }, {
- "tableIdentity" : "SSB.SUMLC_CC_TEST",
- "tableAlias" : "SUMLC_CC_TEST",
- "columnName" : "TO_TIME_COMPOSE_CC",
- "expression" : "concat(\ncast(SUMLC_CC_TEST.year_data as string),'-',\nconcat((case when SUMLC_CC_TEST.month_data<10 then '0' else '' end),cast(SUMLC_CC_TEST.month_data as string)),'-',\nconcat((case when SUMLC_CC_TEST.day_data<10 then '0' else '' end),cast(SUMLC_CC_TEST.day_data as string))\n)",
- "innerExpression" : "concat(\ncast(`SUMLC_CC_TEST`.`YEAR_DATA` as string), '-',\nconcat((case when `SUMLC_CC_TEST`.`MONTH_DATA`<10 then '0' else '' end), cast(`SUMLC_CC_TEST`.`MONTH_DATA` as string)), '-',\nconcat((case when `SUMLC_CC_TEST`.`DAY_DATA`<10 then '0' else '' end), cast(`SUMLC_CC_TEST`.`DAY_DATA` as string))\n)",
- "datatype" : "VARCHAR",
- "comment" : null,
- "rec_uuid" : null
- } ],
- "canvas" : {
- "coordinate" : {
- "SUMLC_CC_TEST" : {
- "x" : 463.5555691189238,
- "y" : 122.99999660915798,
- "width" : 200.0,
- "height" : 466.66666666666663
- }
- },
- "zoom" : 9.0
- },
- "multi_partition_desc" : null,
- "multi_partition_key_mapping" : null,
- "fusion_id" : null
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUMLC_CC_TEST.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUMLC_CC_TEST.json
deleted file mode 100644
index 3c2944ac01..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUMLC_CC_TEST.json
+++ /dev/null
@@ -1,58 +0,0 @@
-{
- "uuid" : "ff1e6d9d-fb31-0269-2547-a6b79cfbe798",
- "last_modified" : 0,
- "create_time" : 1667543492928,
- "version" : "4.0.0.0",
- "name" : "SUMLC_CC_TEST",
- "columns" : [ {
- "id" : "1",
- "name" : "DATE_DATA",
- "datatype" : "integer",
- "case_sensitive_name" : "date_data"
- }, {
- "id" : "2",
- "name" : "YEAR_DATA",
- "datatype" : "integer",
- "case_sensitive_name" : "year_data"
- }, {
- "id" : "3",
- "name" : "MONTH_DATA",
- "datatype" : "integer",
- "case_sensitive_name" : "month_data"
- }, {
- "id" : "4",
- "name" : "DAY_DATA",
- "datatype" : "integer",
- "case_sensitive_name" : "day_data"
- }, {
- "id" : "5",
- "name" : "ACCOUNT",
- "datatype" : "varchar(4096)",
- "case_sensitive_name" : "account"
- }, {
- "id" : "6",
- "name" : "INT_DATA",
- "datatype" : "integer",
- "case_sensitive_name" : "int_data"
- } ],
- "source_type" : 9,
- "table_type" : "MANAGED",
- "top" : false,
- "increment_loading" : false,
- "last_snapshot_path" : null,
- "last_snapshot_size" : 0,
- "snapshot_last_modified" : 0,
- "query_hit_count" : 0,
- "partition_column" : null,
- "snapshot_partitions" : { },
- "snapshot_partitions_info" : { },
- "snapshot_total_rows" : 0,
- "snapshot_partition_col" : null,
- "selected_snapshot_partition_col" : null,
- "temp_snapshot_path" : null,
- "snapshot_has_broken" : false,
- "database" : "SSB",
- "transactional" : false,
- "rangePartition" : false,
- "partition_desc" : null
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUM_LC_NULL_TBL.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUM_LC_NULL_TBL.json
deleted file mode 100644
index be31cb96db..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUM_LC_NULL_TBL.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
- "uuid" : "6d164a7b-de25-e326-32f3-92103e53cf7e",
- "last_modified" : 0,
- "create_time" : 1668136653291,
- "version" : "4.0.0.0",
- "name" : "SUM_LC_NULL_TBL",
- "columns" : [ {
- "id" : "1",
- "name" : "ACCOUNT1",
- "datatype" : "char(1)",
- "case_sensitive_name" : "account1"
- }, {
- "id" : "2",
- "name" : "ACCOUNT2",
- "datatype" : "varchar(52)",
- "case_sensitive_name" : "account2"
- }, {
- "id" : "3",
- "name" : "PART_COL",
- "datatype" : "date",
- "case_sensitive_name" : "part_col"
- }, {
- "id" : "4",
- "name" : "BALANCE1",
- "datatype" : "double",
- "case_sensitive_name" : "balance1"
- }, {
- "id" : "5",
- "name" : "SUM_DATE1",
- "datatype" : "varchar(1024)",
- "case_sensitive_name" : "sum_date1"
- }, {
- "id" : "6",
- "name" : "DATA_NULL",
- "datatype" : "double",
- "case_sensitive_name" : "data_null"
- }, {
- "id" : "7",
- "name" : "DATA_DECIMAL",
- "datatype" : "decimal(10,6)",
- "case_sensitive_name" : "data_decimal"
- } ],
- "source_type" : 9,
- "table_type" : "MANAGED",
- "top" : false,
- "increment_loading" : false,
- "last_snapshot_path" : null,
- "last_snapshot_size" : 0,
- "snapshot_last_modified" : 0,
- "query_hit_count" : 0,
- "partition_column" : null,
- "snapshot_partitions" : { },
- "snapshot_partitions_info" : { },
- "snapshot_total_rows" : 0,
- "snapshot_partition_col" : null,
- "selected_snapshot_partition_col" : null,
- "temp_snapshot_path" : null,
- "snapshot_has_broken" : false,
- "database" : "SSB",
- "transactional" : false,
- "rangePartition" : false,
- "partition_desc" : null
-}
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUM_LC_TB.json b/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUM_LC_TB.json
deleted file mode 100644
index 4148c52c21..0000000000
--- a/src/kylin-it/src/test/resources/ut_meta/sum_lc/metadata/sum_lc/table/SSB.SUM_LC_TB.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
- "uuid" : "6c979409-290a-2deb-dc53-c100ac070ab9",
- "last_modified" : 0,
- "create_time" : 1667828032179,
- "version" : "4.0.0.0",
- "name" : "SUM_LC_TB",
- "columns" : [ {
- "id" : "1",
- "name" : "ACCOUNT",
- "datatype" : "char(1)",
- "case_sensitive_name" : "account"
- }, {
- "id" : "2",
- "name" : "BANK_NAME",
- "datatype" : "varchar(52)",
- "case_sensitive_name" : "bank_name"
- }, {
- "id" : "3",
- "name" : "PART_COL",
- "datatype" : "date",
- "case_sensitive_name" : "part_col"
- }, {
- "id" : "4",
- "name" : "BALANCE",
- "datatype" : "integer",
- "case_sensitive_name" : "balance"
- }, {
- "id" : "5",
- "name" : "SUM_DATE",
- "datatype" : "timestamp",
- "case_sensitive_name" : "sum_date"
- } ],
- "source_type" : 9,
- "table_type" : "MANAGED",
- "top" : false,
- "increment_loading" : false,
- "last_snapshot_path" : null,
- "last_snapshot_size" : 0,
- "snapshot_last_modified" : 0,
- "query_hit_count" : 0,
- "partition_column" : null,
- "snapshot_partitions" : { },
- "snapshot_partitions_info" : { },
- "snapshot_total_rows" : 0,
- "snapshot_partition_col" : null,
- "selected_snapshot_partition_col" : null,
- "temp_snapshot_path" : null,
- "snapshot_has_broken" : false,
- "database" : "SSB",
- "transactional" : false,
- "rangePartition" : false,
- "partition_desc" : null
-}
\ No newline at end of file
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 12bb43bf49..d83c15c751 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -3545,10 +3545,6 @@ public class ModelService extends AbstractModelService implements TableModelSupp
validateFusionModelDimension(modelRequest);
NDataModel model = semanticUpdater.convertToDataModel(modelRequest);
- for(NDataModel.Measure measure : model.getAllMeasures()) {
- measure.getFunction().init(model);
- }
-
if (modelRequest.getPartitionDesc() != null
&& !KylinConfig.getInstanceFromEnv().isUseBigIntAsTimestampForPartitionColumn()) {
PartitionDesc partitionDesc = modelRequest.getPartitionDesc();
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 05930f78bc..eb859e0ed9 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -5688,21 +5688,4 @@ public class ModelServiceTest extends SourceTestCase {
Assert.assertEquals("DEFAULT.TEST_ORDER.ORDER_ID", colRef1.getCanonicalName());
}
}
-
- @Test
- public void testInitModel() throws IOException {
- String modelRequest =
- "{\"uuid\":null,\"name\":\"sum_lc_null_val_test_clone\",\"owner\":\"ADMIN\",\"project\":\"sum_lc\",\"description\":null,"
- + "\"alias\":\"sum_lc_null_val_test_clone\",\"fact_table\":\"SSB.SUM_LC_NULL_TBL\",\"join_tables\":[],"
- + "\"simplified_dimensions\":[{\"id\":0,\"name\":\"PART_COL\",\"column\":\"SUM_LC_NULL_TBL.PART_COL\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\":null,\"min_value\":null,\"max_value\":null,\"max_length_value\":null,\"min_length_value\":null,\"null_count\":null,\"comment\":null,\"type\":\"date\",\"simple\":null,\"datatype\":\"date\"},"
- + "{\"id\":1,\"name\":\"SUM_DATE1\",\"column\":\"SUM_LC_NULL_TBL.SUM_DATE1\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\":null,\"min_value\":null,\"max_value\":null,\"max_length_value\":null,\"min_length_value\":null,\"null_count\":null,\"comment\":null,\"type\":\"varchar(1024)\",\"simple\":null,\"datatype\":\"varchar(1024)\"},{\"id\":2,\"name\":\"ACCOUNT1\",\"column\":\"SUM_LC_NULL_TBL.ACCOUNT1\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardin [...]
- + "{\"id\":6,\"name\":\"ACCOUNT2\",\"column\":\"SUM_LC_NULL_TBL.ACCOUNT2\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\":null,\"min_value\":null,\"max_value\":null,\"max_length_value\":null,\"min_length_value\":null,\"null_count\":null,\"comment\":null,\"type\":\"varchar(52)\",\"simple\":null,\"datatype\":\"varchar(52)\"}],"
- + "\"simplified_measures\":[{\"id\":100000,\"expression\":\"COUNT\",\"name\":\"COUNT_ALL\",\"return_type\":\"bigint\",\"parameter_value\":[{\"type\":\"constant\",\"value\":\"1\"}],\"converted_columns\":[],\"column\":null,\"comment\":null},{\"id\":100001,\"expression\":\"SUM_LC\",\"name\":\"sumlc_double_null\",\"return_type\":\"double\",\"parameter_value\":[{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.DATA_NULL\"},{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TB [...]
- + "\"converted_columns\":[],\"column\":null,\"comment\":\"\"},{\"id\":100002,\"expression\":\"SUM_LC\",\"name\":\"sumlc_decimal_null\",\"return_type\":\"decimal(20,6)\",\"parameter_value\":[{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.DATA_DECIMAL\"},{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.SUM_DATE1\"}],\"converted_columns\":[],\"column\":null,\"comment\":\"\"},{\"name\":\"abc\",\"expression\":\"SUM_LC\",\"return_type\":\"\",\"comment\":\"\",\"paramet [...]
- + "\"computed_columns\":[],\"last_modified\":1668402813791,\"filter_condition\":\"\",\"partition_desc\":null,\"multi_partition_desc\":null,\"management_type\":\"MODEL_BASED\",\"with_second_storage\":false,\"second_storage_size\":0,\"canvas\":{\"coordinate\":{\"SUM_LC_NULL_TBL\":{\"x\":462.44444105360253,\"y\":108.66667005750864,\"width\":200,\"height\":486.66666666666663,\"isSpread\":true}},\"zoom\":9,\"marginClient\":{\"left\":0,\"top\":0}},\"available_indexes_co [...]
- + "{\"column\":\"SUM_LC_NULL_TBL.DATA_NULL\",\"name\":\"DATA_NULL\",\"datatype\":\"double\"},{\"column\":\"SUM_LC_NULL_TBL.DATA_DECIMAL\",\"name\":\"DATA_DECIMAL\",\"datatype\":\"decimal(10,6)\"}]}";
- ModelRequest request = JsonUtil.readValue(modelRequest, ModelRequest.class);
- Assert.assertThrows(KylinException.class, () -> modelService.checkBeforeModelSave(request));
- }
-
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java b/src/query-common/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
index b8fa24d009..18ec915c90 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
@@ -29,7 +29,7 @@ import org.apache.kylin.query.IQueryTransformer;
*/
public class DefaultQueryTransformer implements IQueryTransformer {
- public static final String S0 = "\\s*";
+ private static final String S0 = "\\s*";
private static final String SM = "\\s+";
private static final String ONE = "1";
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
index a1de745089..81439609e9 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
@@ -19,7 +19,6 @@
package org.apache.kylin.query.engine;
import static org.apache.kylin.query.relnode.OLAPContext.clearThreadLocalContexts;
-import static org.apache.kylin.query.util.DefaultQueryTransformer.S0;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -29,7 +28,6 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.regex.Pattern;
import org.apache.calcite.avatica.ColumnMetaData;
import org.apache.calcite.prepare.CalcitePrepareImpl;
@@ -76,10 +74,6 @@ public class QueryRoutingEngine {
public static final String SPARK_MEM_LIMIT_EXCEEDED = "Container killed by YARN for exceeding memory limits";
public static final String SPARK_JOB_FAILED = "Job aborted due to stage failure";
- private static final Pattern PTN_SUM_LC = Pattern.compile(
- S0 + "\\bSUM_LC" + S0 + "[(]" + S0 + ".*\\.?.*" + S0 + "[,]" + S0 + ".*\\.?.*" + S0 + "[)]" + S0,
- Pattern.CASE_INSENSITIVE);
-
public QueryResult queryWithSqlMassage(QueryParams queryParams) throws Exception {
QueryContext.current().setAclInfo(queryParams.getAclInfo());
KylinConfig projectKylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
@@ -88,7 +82,7 @@ public class QueryRoutingEngine {
queryParams.setDefaultSchema(queryExec.getDefaultSchemaName());
if (queryParams.isForcedToPushDown()) {
- checkContainsSumLC(queryParams);
+ checkContainsSumLC(queryParams, null);
return pushDownQuery(null, queryParams);
}
@@ -136,6 +130,7 @@ public class QueryRoutingEngine {
if (cause instanceof SQLException && cause.getCause() instanceof KylinException) {
throw (SQLException) cause;
}
+ checkContainsSumLC(queryParams, e);
if (shouldPushdown(cause, queryParams)) {
return pushDownQuery((SQLException) cause, queryParams);
} else {
@@ -154,6 +149,7 @@ public class QueryRoutingEngine {
}
}
}
+ checkContainsSumLC(queryParams, e);
if (shouldPushdown(e, queryParams)) {
return pushDownQuery(e, queryParams);
} else {
@@ -175,14 +171,18 @@ public class QueryRoutingEngine {
return false;
}
- private void checkContainsSumLC(QueryParams queryParams) {
- if (PTN_SUM_LC.matcher(queryParams.getSql()).find()) {
- String message = "sum_lc() function now is not supported by other query engine";
- throw new NotSupportedSQLException(message);
+ private void checkContainsSumLC(QueryParams queryParams, Throwable t) {
+ if (queryParams.getSql().contains("sum_lc")) {
+ String message = "There is no aggregate index to answer this query, sum_lc() function now is not supported by other query engine";
+ if (t != null) {
+ throw new NotSupportedSQLException(message, t);
+ } else {
+ throw new NotSupportedSQLException(message);
+ }
}
}
- protected boolean shouldPushdown(Throwable e, QueryParams queryParams) {
+ private boolean shouldPushdown(Throwable e, QueryParams queryParams) {
if (queryParams.isForcedToIndex()) {
return false;
}
@@ -199,10 +199,6 @@ public class QueryRoutingEngine {
return checkBigQueryPushDown(queryParams);
}
- if (PTN_SUM_LC.matcher(queryParams.getSql()).find()) {
- return false;
- }
-
return e instanceof SQLException && !e.getMessage().contains(SPARK_MEM_LIMIT_EXCEEDED);
}
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
index 22af9a7c02..7599095bf6 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
@@ -308,7 +308,7 @@ public class QueryRoutingEngineTest extends NLocalFileMetadataTestCase {
@Test
public void testQueryPushDownWithSumLC() {
- final String sql = "select sUm_Lc \r\n ( \r\n \"success_table_2\".\"column\", \r\n dateColumn) from success_table_2";
+ final String sql = "select sum_lc(column, dateColumn) from success_table_2";
final String project = "default";
KylinConfig kylinconfig = KylinConfig.getInstanceFromEnv();
QueryParams queryParams = new QueryParams();
@@ -320,20 +320,4 @@ public class QueryRoutingEngineTest extends NLocalFileMetadataTestCase {
Assert.assertThrows(NotSupportedSQLException.class, () -> queryRoutingEngine.queryWithSqlMassage(queryParams));
}
-
- @Test
- public void testShouldPushDown() {
- final String sql = "select sUm_Lc \r\n ( \r\n \"success_table_2\".\"column\", \r\n dateColumn) from success_table_2";
- final String project = "default";
- KylinConfig kylinconfig = KylinConfig.getInstanceFromEnv();
- QueryParams queryParams = new QueryParams();
- queryParams.setProject(project);
- queryParams.setSql(sql);
- queryParams.setKylinConfig(kylinconfig);
- queryParams.setSelect(true);
- queryParams.setForcedToPushDown(true);
-
- boolean shouldPushDown = queryRoutingEngine.shouldPushdown(new RuntimeException(), queryParams);
- Assert.assertEquals(false, shouldPushDown);
- }
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionUtils.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionUtils.scala
index ebf68446ad..321bece62c 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionUtils.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionUtils.scala
@@ -139,11 +139,7 @@ object ExpressionUtils {
val arrayBytes = bytes.asInstanceOf[Array[Byte]]
val codec = SumLCUtil.getNumericNullSafeSerializerByDataType(DataType.fromJson(wrapDataType.toString))
val counter = SumLCUtil.decodeToSumLCCounter(arrayBytes, codec)
- if (counter == null) {
- null
- } else {
- counter.getSumLC
- }
+ counter.getSumLC
}
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
index d7755bcaaf..ea28802be5 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
@@ -828,8 +828,6 @@ case class SumLCDecode(bytes: Expression, wrapDataTypeExpr: Expression) extends
override def prettyName: String = "sum_lc_decode"
- override def nullable: Boolean = true
-
override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Expression = {
val newChildren = Seq(newLeft, newRight)
super.legacyWithNewChildren(newChildren)
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/SumLC.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/SumLC.scala
index 6472b742dd..bb68841e01 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/SumLC.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/SumLC.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.udaf
+import com.esotericsoftware.kryo.KryoException
import com.esotericsoftware.kryo.io.{Input, KryoDataInput, KryoDataOutput, Output}
import org.apache.kylin.common.util.DateFormat
import org.apache.kylin.measure.sumlc.SumLCCounter
@@ -42,9 +43,6 @@ sealed abstract class BaseSumLC(wrapDataType: DataType,
mutableAggBufferOffset: Int = 0,
inputAggBufferOffset: Int = 0)
extends TypedImperativeAggregate[SumLCCounter] with Serializable with Logging {
- private val INIT_ARRAY_BUFFER_LENGTH = 32;
- private val MAX_ARRAY_BUFFER_LENGTH = 256;
-
lazy val serializer: NullSafeValueSerializer = SumLCUtil.getNumericNullSafeSerializerByDataType(wrapDataType)
override def prettyName: String = "sum_lc"
@@ -54,38 +52,49 @@ sealed abstract class BaseSumLC(wrapDataType: DataType,
case BinaryType =>
serialize(buffer)
case DecimalType() =>
- if (buffer == null || buffer.getSumLC == null) return null;
- Decimal.fromDecimal(buffer.getSumLC.asInstanceOf[java.math.BigDecimal])
+ if (buffer.getSumLC != null) {
+ Decimal.fromDecimal(buffer.getSumLC.asInstanceOf[java.math.BigDecimal])
+ } else {
+ Decimal.ZERO
+ }
case _ =>
- if (buffer == null) return null;
buffer.getSumLC
}
}
- override def createAggregationBuffer(): SumLCCounter = null
+ override def createAggregationBuffer(): SumLCCounter = new SumLCCounter()
override def merge(buffer: SumLCCounter, input: SumLCCounter): SumLCCounter = {
SumLCCounter.merge(buffer, input)
}
override def serialize(buffer: SumLCCounter): Array[Byte] = {
- if (buffer == null) {
- Array.empty[Byte]
- } else {
- val output: Output = new Output(INIT_ARRAY_BUFFER_LENGTH, MAX_ARRAY_BUFFER_LENGTH)
- try {
- val dataOutput = new KryoDataOutput(output)
- serializer.serialize(dataOutput, buffer.getSumLC)
- dataOutput.writeLong(buffer.getTimestamp)
+ val array: Array[Byte] = new Array[Byte](1024 * 1024)
+ val output: Output = new Output(array)
+ serialize(buffer, array, output)
+ }
+
+ private def serialize(buffer: SumLCCounter, array: Array[Byte], output: Output): Array[Byte] = {
+ try {
+ if (buffer == null) {
+ Array.empty[Byte]
+ } else {
+ output.clear()
+ val out = new KryoDataOutput(output)
+ serializer.serialize(out, buffer.getSumLC)
+ out.writeLong(buffer.getTimestamp)
val mark = output.position()
- val bufferArray = output.getBuffer
- bufferArray.slice(0, mark)
- } catch {
- case th: Throwable =>
- throw th
- } finally if (output != null) {
output.close()
+ array.slice(0, mark)
}
+ } catch {
+ case th: KryoException if th.getMessage.contains("Buffer overflow") =>
+ logWarning(s"Resize buffer size to ${array.length * 2}")
+ val updateArray = new Array[Byte](array.length * 2)
+ output.setBuffer(updateArray)
+ serialize(buffer, updateArray, output)
+ case th =>
+ throw th
}
}
@@ -93,18 +102,9 @@ sealed abstract class BaseSumLC(wrapDataType: DataType,
SumLCUtil.decodeToSumLCCounter(bytes, serializer)
}
- override def nullable: Boolean = true
+ override def nullable: Boolean = false
override def dataType: DataType = outputDataType
-
- protected def sumLCUpdateInternal(buffer: SumLCCounter, columnVal: Number, timestampVal: Long): SumLCCounter = {
- if (buffer == null) {
- new SumLCCounter(columnVal, timestampVal)
- } else {
- buffer.update(columnVal, timestampVal)
- buffer
- }
- }
}
case class EncodeSumLC(
@@ -124,14 +124,9 @@ case class EncodeSumLC(
case _ =>
columnEvalVal.asInstanceOf[Number]
}
- val dateEvalVal = dateCol.eval(input)
- if (dateEvalVal == null || dateEvalVal.toString.toUpperCase().equals("NULL")) {
- buffer
- } else {
- val dateValStr = String.valueOf(dateEvalVal).trim
- val timestampVal = DateFormat.stringToMillis(dateValStr)
- sumLCUpdateInternal(buffer, columnVal, timestampVal)
- }
+ val dateValStr = String.valueOf(dateCol.eval(input)).trim
+ val timestampVal = DateFormat.stringToMillis(dateValStr)
+ SumLCCounter.merge(buffer, columnVal, timestampVal)
}
override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
@@ -154,14 +149,8 @@ case class ReuseSumLC(measure: Expression,
extends BaseSumLC(wrapDataType, outputDataType, mutableAggBufferOffset, inputAggBufferOffset) {
override def update(buffer: SumLCCounter, input: InternalRow): SumLCCounter = {
- val valAndTsPair = SumLCUtil.decodeToValAndTs(measure.eval(input).asInstanceOf[Array[Byte]], serializer)
- if (valAndTsPair == null) {
- buffer
- } else {
- val columnVal = valAndTsPair._1
- val timestampVal = valAndTsPair._2
- sumLCUpdateInternal(buffer, columnVal, timestampVal)
- }
+ val evalCounter = deserialize(measure.eval(input).asInstanceOf[Array[Byte]])
+ SumLCCounter.merge(buffer, evalCounter)
}
override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
@@ -179,23 +168,14 @@ case class ReuseSumLC(measure: Expression,
object SumLCUtil extends Logging {
- def decodeToValAndTs(bytes: Array[Byte], codec: NullSafeValueSerializer): (Number, Long) = {
- if (bytes.isEmpty) {
- null
- } else {
+ def decodeToSumLCCounter(bytes: Array[Byte], codec: NullSafeValueSerializer): SumLCCounter = {
+ if (bytes.nonEmpty) {
val in = new KryoDataInput(new Input(bytes))
val sumLC = codec.deserialize(in).asInstanceOf[Number]
val timestamp = in.readLong()
- (sumLC, timestamp)
- }
- }
-
- def decodeToSumLCCounter(bytes: Array[Byte], codec: NullSafeValueSerializer): SumLCCounter = {
- val valAndTsPair = decodeToValAndTs(bytes, codec)
- if (valAndTsPair == null) {
- null
+ new SumLCCounter(sumLC, timestamp)
} else {
- new SumLCCounter(valAndTsPair._1, valAndTsPair._2)
+ new SumLCCounter()
}
}