You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/05 10:21:05 UTC
[kylin] 13/22: KYLIN-5318 adjust CC names Adjust dimensions measure and filter condition simultaneously
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8b31539dab85375937597d7bfff8ef17a89a27bc
Author: Jiale He <35...@users.noreply.github.com>
AuthorDate: Fri Oct 14 14:08:25 2022 +0800
KYLIN-5318 adjust CC names Adjust dimensions measure and filter condition simultaneously
---
.../resources/kylin_error_msg_conf_cn.properties | 2 +-
.../resources/kylin_error_msg_conf_en.properties | 2 +-
.../metadata/model/util/ComputedColumnUtil.java | 8 +-
.../apache/kylin/rest/service/ModelService.java | 55 ++++++++++-
.../kylin/rest/service/ModelServiceTest.java | 102 ++++++++++++++++++---
5 files changed, 149 insertions(+), 20 deletions(-)
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 327c28d18d..759514ee40 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -98,7 +98,7 @@ KE-010031201=因为查询结果行数超过最大值 ”%s”，无法完成查
KE-010031202=SQL 语法或格式异常，请检查并修正后重试。
## 100102XX computed column
-KE-010010201=模型中定义的可计算列的名和表达式与其它模型存在冲突，请修改名称以保持一致，或使用其他的表达式。
+KE-010010201=模型中定义的可计算列的名和表达式与其它模型存在冲突。
KE-010010202=重复的可计算列名，名为 “%s” 表达式为 “%s” 的可计算列，与模型 “%s” 中的可计算列名存在冲突。
KE-010010203=重复的可计算列表达式，名为 “%s” 表达式为 “%s” 的可计算列，与模型 “%s” 中的可计算列表达式存在冲突。
KE-010010204=名为 “%s” 表达式为 “%s” 的可计算列，与项目中名为 “%s” 表达式为 “%s” 的可计算列表达式存在冲突，将当前可计算列重命名为 “%s” 。
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index 5a5f9ea5fb..02ecccdb25 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -96,7 +96,7 @@ KE-010031201=Can't get query result, as the rows of query result exceeds the max
KE-010031202=SQL syntax or format is abnormal, please check and fix and try again.
## 100102XX computed column
-KE-010010201=The name and expression of the computed column defined in the model conflict with other models. Please modify the name to be consistent, or use another expression.
+KE-010010201=The name and expression of the computed column defined in the model conflict with other models.
KE-010010202=Duplicate computed column name, defined computed column named "%s" with expression "%s", conflicts with a computed column name in model "%s".
KE-010010203=Duplicate computed column expression, defined computed column named "%s" with expression "%s", conflicts with a computed column expression in model "%s".
KE-010010204=Defined computed column named "%s" with expression "%s" is inconsistent with the name of the computed column named "%s" with the expression "%s" in the project. Renamed to "%s".
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java
index b07c7cf5b0..ead8a3e8fd 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java
@@ -626,10 +626,10 @@ public class ComputedColumnUtil {
return exceptionList;
}
- public Pair<List<ComputedColumnDesc>, List<KylinException>> getAdjustedCCList(
+ public Pair<List<ComputedColumnDesc>, List<CCConflictDetail>> getAdjustedCCList(
List<ComputedColumnDesc> inputCCDescList) {
List<ComputedColumnDesc> resultCCDescList = Lists.newArrayList();
- List<KylinException> adjustExceptionList = Lists.newArrayList();
+ List<CCConflictDetail> adjustDetails = Lists.newArrayList();
for (ComputedColumnDesc ccDesc : inputCCDescList) {
for (CCConflictDetail detail : this.sameExprDiffNameDetails) {
@@ -638,13 +638,13 @@ public class ComputedColumnUtil {
if (newCC.equals(ccDesc)) {
logger.info("adjust cc name {} to {}", newCC.getColumnName(), existingCC.getColumnName());
ccDesc.setColumnName(existingCC.getColumnName());
- adjustExceptionList.add(detail.getAdjustKylinException());
+ adjustDetails.add(detail);
break;
}
}
resultCCDescList.add(ccDesc);
}
- return Pair.newPair(resultCCDescList, adjustExceptionList);
+ return Pair.newPair(resultCCDescList, adjustDetails);
}
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 7082c4890b..7fcf9f610c 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -70,6 +70,7 @@ import static org.apache.kylin.job.execution.JobTypeEnum.INC_BUILD;
import static org.apache.kylin.job.execution.JobTypeEnum.INDEX_BUILD;
import static org.apache.kylin.job.execution.JobTypeEnum.INDEX_MERGE;
import static org.apache.kylin.job.execution.JobTypeEnum.INDEX_REFRESH;
+import static org.apache.kylin.metadata.model.FunctionDesc.PARAMETER_TYPE_COLUMN;
import java.io.IOException;
import java.math.BigDecimal;
@@ -190,6 +191,7 @@ import org.apache.kylin.metadata.model.UpdateImpact;
import org.apache.kylin.metadata.model.VolatileRange;
import org.apache.kylin.metadata.model.schema.AffectedModelContext;
import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.kylin.metadata.model.util.MultiPartitionUtil;
import org.apache.kylin.metadata.model.util.scd2.SCD2CondChecker;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
@@ -4242,8 +4244,59 @@ public class ModelService extends AbstractModelService implements TableModelSupp
List<ComputedColumnDesc> inputCCDescList = Lists.newArrayList(modelRequest.getComputedColumnDescs());
// deal with conflicts
val pair = ccConflictInfo.getAdjustedCCList(inputCCDescList);
+ val adjustExceptions = pair.getSecond().stream() //
+ .map(ComputedColumnUtil.CCConflictDetail::getAdjustKylinException).collect(Collectors.toList());
+ ModelRequest resultModelRequest = adjustModelRequestCCName(modelRequest, pair);
+
+ return Pair.newPair(resultModelRequest, handleOnConflictResponse(adjustExceptions));
+ }
+
+ public ModelRequest adjustModelRequestCCName(ModelRequest modelRequest,
+ Pair<List<ComputedColumnDesc>, List<ComputedColumnUtil.CCConflictDetail>> pair) {
+ val adjustDetails = pair.getSecond();
+ // adjust cc name
modelRequest.setComputedColumnDescs(pair.getFirst());
- return Pair.newPair(modelRequest, handleOnConflictResponse(pair.getSecond()));
+
+ val dimensions = modelRequest.getSimplifiedDimensions();
+ val measures = modelRequest.getSimplifiedMeasures();
+ for (val detail : adjustDetails) {
+ String newCCFullName = detail.getNewCC().getFullName();
+ String existingCCFullName = detail.getExistingCC().getFullName();
+
+ // adjust dimensions
+ dimensions.stream() //
+ .filter(NDataModel.NamedColumn::isExist) //
+ // column equals
+ .filter(d -> StringUtils.equalsIgnoreCase(d.getAliasDotColumn(), newCCFullName))
+ .forEach(d -> d.setAliasDotColumn(existingCCFullName));
+
+ // adjust measures
+ measures.forEach(m -> m.getParameterValue().stream() //
+ // type = column
+ .filter(pr -> StringUtils.equalsIgnoreCase(pr.getType(), PARAMETER_TYPE_COLUMN))
+ // value equals
+ .filter(pr -> StringUtils.equalsIgnoreCase(pr.getValue(), newCCFullName))
+ .forEach(pr -> pr.setValue(existingCCFullName)));
+ }
+
+ // adjust filter condition
+ String filterCondition = modelRequest.getFilterCondition();
+ if (StringUtils.isEmpty(filterCondition)) {
+ return modelRequest;
+ }
+ for (val detail : adjustDetails) {
+ String newCCFullName = detail.getNewCC().getFullName();
+ String existingCCFullName = detail.getExistingCC().getFullName();
+ if (StringUtils.containsIgnoreCase(filterCondition, newCCFullName)) {
+ filterCondition = replaceAllIgnoreCase(filterCondition, newCCFullName, existingCCFullName);
+ }
+ }
+ modelRequest.setFilterCondition(filterCondition);
+ return modelRequest;
+ }
+
+ public String replaceAllIgnoreCase(String input, String regex, String replacement) {
+ return Pattern.compile(regex, Pattern.CASE_INSENSITIVE).matcher(input).replaceAll(replacement);
}
public ComputedColumnConflictResponse handleOnConflictResponse(List<KylinException> exceptionList) {
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index adad3571f6..e9dc0f22c3 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -5276,8 +5276,8 @@ public class ModelServiceTest extends SourceTestCase {
testCheckCCConflictAllExprConflict(originRequest);
testCheckCCConflictExprAndNameConflict(originRequest);
testCheckCCConflictExprAndNameConflict2(originRequest);
- testCheckCCConflictAdjust(originRequest);
testNoCCConflict(originRequest);
+ testCheckCCConflictAdjust(originRequest);
}
private void testCheckCCConflictAllExprConflict(ModelRequest originRequest) {
@@ -5369,18 +5369,78 @@ public class ModelServiceTest extends SourceTestCase {
}
private void testCheckCCConflictAdjust(ModelRequest originRequest) {
- val ccList = Lists.newArrayList(//
- getComputedColumnDesc("CC_1", "CUSTOMER.C_NAME +'USA'", "DOUBLE"),
- getComputedColumnDesc("CC_LTAX", "LINEORDER.LO_TAX + 1", "BIGINT"));
- originRequest.setComputedColumnDescs(ccList);
- originRequest.setComputedColumnNameAutoAdjust(true);
- val pair = modelService.checkCCConflict(originRequest);
- val details = pair.getSecond().getConflictDetails();
- Assert.assertEquals(1, details.size());
- Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getErrorCode().getCode(),
- details.get(0).getDetailCode());
- Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getMsg("CC_1", "CUSTOMER.C_NAME +'USA'", "CC_CNAME",
- "CUSTOMER.C_NAME +'USA'", "CC_CNAME"), details.get(0).getDetailMsg());
+ {
+ val ccList = Lists.newArrayList(//
+ getComputedColumnDesc("CC_1", "CUSTOMER.C_NAME +'USA'", "DOUBLE"),
+ getComputedColumnDesc("CC_LTAX", "LINEORDER.LO_TAX + 1", "BIGINT"));
+ originRequest.setComputedColumnDescs(ccList);
+ originRequest.setComputedColumnNameAutoAdjust(true);
+ val pair = modelService.checkCCConflict(originRequest);
+ val details = pair.getSecond().getConflictDetails();
+ Assert.assertEquals(1, details.size());
+ Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getErrorCode().getCode(),
+ details.get(0).getDetailCode());
+ Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getMsg("CC_1", "CUSTOMER.C_NAME +'USA'",
+ "CC_CNAME", "CUSTOMER.C_NAME +'USA'", "CC_CNAME"), details.get(0).getDetailMsg());
+ }
+
+ {
+ val ccList = Lists.newArrayList(//
+ getComputedColumnDesc("CC_1", "CUSTOMER.C_NAME +'USA'", "DOUBLE"),
+ getComputedColumnDesc("CC_LTAX", "LINEORDER.LO_TAX + 1", "BIGINT"));
+ originRequest.setComputedColumnDescs(ccList);
+ originRequest.setComputedColumnNameAutoAdjust(true);
+ originRequest.setFilterCondition("LINEORDER.LO_TAX = 'Kylin' or LINEORDER.LO_TAX = 'Kylin2'");
+ val pair = modelService.checkCCConflict(originRequest);
+ val details = pair.getSecond().getConflictDetails();
+ Assert.assertEquals(1, details.size());
+ Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getErrorCode().getCode(),
+ details.get(0).getDetailCode());
+ Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getMsg("CC_1", "CUSTOMER.C_NAME +'USA'",
+ "CC_CNAME", "CUSTOMER.C_NAME +'USA'", "CC_CNAME"), details.get(0).getDetailMsg());
+ Assert.assertEquals("LINEORDER.LO_TAX = 'Kylin' or LINEORDER.LO_TAX = 'Kylin2'",
+ pair.getFirst().getFilterCondition());
+ }
+
+ {
+ val dimList = Lists.newArrayList(getNamedColumn("CC_1", "LINEORDER.CC_1"));
+ val measureList = Lists.newArrayList(//
+ getSimplifiedMeasure("cc_count", "COUNT", "column", "LINEORDER.CC_1"),
+ getSimplifiedMeasure("COUNT_ALL", "COUNT", "constant", "1"));
+ val ccList = Lists.newArrayList(//
+ getComputedColumnDesc("CC_1", "CUSTOMER.C_NAME +'USA'", "DOUBLE"),
+ getComputedColumnDesc("CC_LTAX", "LINEORDER.LO_TAX + 1", "BIGINT"));
+ originRequest.setComputedColumnDescs(ccList);
+ originRequest.setComputedColumnNameAutoAdjust(true);
+ originRequest.setSimplifiedDimensions(dimList);
+ originRequest.setSimplifiedMeasures(measureList);
+ originRequest.setFilterCondition("LINEORDER.Cc_1 = 'Kylin' or LINEORDER.cC_1 = 'Kylin2'");
+ val pair = modelService.checkCCConflict(originRequest);
+ val details = pair.getSecond().getConflictDetails();
+ Assert.assertEquals(1, details.size());
+ Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getErrorCode().getCode(),
+ details.get(0).getDetailCode());
+ Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getMsg("CC_1", "CUSTOMER.C_NAME +'USA'",
+ "CC_CNAME", "CUSTOMER.C_NAME +'USA'", "CC_CNAME"), details.get(0).getDetailMsg());
+
+ ModelRequest modelRequest = pair.getFirst();
+ val simplifiedDimensions = modelRequest.getSimplifiedDimensions();
+ Assert.assertEquals(1, simplifiedDimensions.size());
+ Assert.assertEquals("LINEORDER.CC_CNAME", simplifiedDimensions.get(0).getAliasDotColumn());
+ Assert.assertEquals("CC_1", simplifiedDimensions.get(0).getName());
+
+ List<SimplifiedMeasure> simplifiedMeasures = modelRequest.getSimplifiedMeasures();
+ Assert.assertEquals(2, simplifiedMeasures.size());
+ simplifiedMeasures = simplifiedMeasures.stream().filter(measure -> measure.getName().equals("cc_count"))
+ .collect(Collectors.toList());
+ Assert.assertEquals(1, simplifiedMeasures.size());
+ Assert.assertEquals("COUNT", simplifiedMeasures.get(0).getExpression());
+ Assert.assertEquals("column", simplifiedMeasures.get(0).getParameterValue().get(0).getType());
+ Assert.assertEquals("LINEORDER.CC_CNAME", simplifiedMeasures.get(0).getParameterValue().get(0).getValue());
+
+ Assert.assertEquals("LINEORDER.CC_CNAME = 'Kylin' or LINEORDER.CC_CNAME = 'Kylin2'",
+ modelRequest.getFilterCondition());
+ }
}
private void testNoCCConflict(ModelRequest originRequest) {
@@ -5438,4 +5498,20 @@ public class ModelServiceTest extends SourceTestCase {
return ccDesc;
}
+ private NamedColumn getNamedColumn(String name, String aliasDotName) {
+ NamedColumn namedColumn = new NamedColumn();
+ namedColumn.setName(name);
+ namedColumn.setAliasDotColumn(aliasDotName);
+ namedColumn.setStatus(NDataModel.ColumnStatus.DIMENSION);
+ return namedColumn;
+ }
+
+ private SimplifiedMeasure getSimplifiedMeasure(String name, String expr, String type, String value) {
+ ParameterResponse parameterResponse = new ParameterResponse(type, value);
+ SimplifiedMeasure measure = new SimplifiedMeasure();
+ measure.setName(name);
+ measure.setExpression(expr);
+ measure.setParameterValue(Lists.newArrayList(parameterResponse));
+ return measure;
+ }
}