You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/04/16 12:03:31 UTC
(doris) 08/39: [opt](Nereids) prefer slot type to support delete task better (#33559)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 973d94ebccbe5bb36d6de87aac9b39a2bc28dd15
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Mon Apr 15 15:00:36 2024 +0800
[opt](Nereids) prefer slot type to support delete task better (#33559)
---
.../apache/doris/nereids/types/DecimalV2Type.java | 4 +
.../doris/nereids/util/TypeCoercionUtils.java | 96 ++++++++++++++++++++++
.../doris/nereids/util/TypeCoercionUtilsTest.java | 67 +++++++++++++++
3 files changed, 167 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java
index 8a5a199a8d4..c0f023ed739 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java
@@ -173,6 +173,10 @@ public class DecimalV2Type extends FractionalType {
return scale;
}
+ public int getRange() {
+ return precision - scale;
+ }
+
@Override
public DataType conversion() {
if (Config.enable_decimal_conversion && shouldConversion) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
index afcdb30f2dc..214fbc18049 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
@@ -78,6 +78,7 @@ import org.apache.doris.nereids.types.CharType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DateTimeType;
import org.apache.doris.nereids.types.DateTimeV2Type;
+import org.apache.doris.nereids.types.DateType;
import org.apache.doris.nereids.types.DateV2Type;
import org.apache.doris.nereids.types.DecimalV2Type;
import org.apache.doris.nereids.types.DecimalV3Type;
@@ -120,6 +121,7 @@ import java.math.BigInteger;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.function.Function;
import java.util.stream.Collectors;
/**
@@ -933,6 +935,18 @@ public class TypeCoercionUtils {
Optional<DataType> commonType = findWiderTypeForTwoForComparison(
left.getDataType(), right.getDataType(), false);
+
+ if (commonType.isPresent()) {
+ commonType = Optional.of(downgradeDecimalAndDateLikeType(
+ commonType.get(),
+ left,
+ right));
+ commonType = Optional.of(downgradeDecimalAndDateLikeType(
+ commonType.get(),
+ right,
+ left));
+ }
+
if (commonType.isPresent()) {
if (!supportCompare(commonType.get())) {
throw new AnalysisException("data type " + commonType.get()
@@ -982,6 +996,13 @@ public class TypeCoercionUtils {
+ " could not used in InPredicate " + inPredicate.toSql());
}
+ if (optionalCommonType.isPresent()) {
+ optionalCommonType = Optional.of(downgradeDecimalAndDateLikeType(
+ optionalCommonType.get(),
+ inPredicate.getCompareExpr(),
+ inPredicate.getOptions().toArray(new Expression[0])));
+ }
+
return optionalCommonType
.map(commonType -> {
List<Expression> newChildren = inPredicate.children().stream()
@@ -992,6 +1013,81 @@ public class TypeCoercionUtils {
.orElse(inPredicate);
}
+ /**
+ * if the expression like slot vs literal, then we prefer cast literal to slot type to ensure delete task could run.
+ * currently process decimalv2 vs decimalv3, datev1 vs datev2, datetimev1 vs datetimev2.
+ */
+ private static DataType downgradeDecimalAndDateLikeType(
+ DataType commonType,
+ Expression target,
+ Expression... compareExpressions) {
+ if (shouldDowngrade(DecimalV3Type.class, DecimalV2Type.class,
+ commonType, target,
+ d -> ((DecimalV3Type) d).getRange() <= DecimalV2Type.MAX_PRECISION - DecimalV2Type.MAX_SCALE
+ && ((DecimalV3Type) d).getScale() <= DecimalV2Type.MAX_SCALE,
+ o -> o.isLiteral()
+ && (o.getDataType().isDecimalV2Type() || o.getDataType().isDecimalV3Type()),
+ compareExpressions)) {
+ DecimalV3Type decimalV3Type = (DecimalV3Type) commonType;
+ return DecimalV2Type.createDecimalV2Type(decimalV3Type.getPrecision(), decimalV3Type.getScale());
+ }
+ // cast to datev1 for datev1 slot in (datev1 or datev2 literal)
+ if (shouldDowngrade(DateV2Type.class, DateType.class,
+ commonType, target,
+ d -> true,
+ o -> o.isLiteral()
+ && (o.getDataType().isDateType() || o.getDataType().isDateV2Type()),
+ compareExpressions)) {
+ return DateType.INSTANCE;
+ }
+ // cast to datetimev1 for datetimev1 slot in (date like literal) if scale is 0
+ if (shouldDowngrade(DateTimeV2Type.class, DateTimeType.class,
+ commonType, target,
+ d -> ((DateTimeV2Type) d).getScale() == 0,
+ o -> o.isLiteral() && o.getDataType().isDateLikeType(),
+ compareExpressions)) {
+ return DateTimeType.INSTANCE;
+ }
+ return commonType;
+ }
+
+ /**
+ * check should downgrade from commonTypeClazz to targetTypeClazz.
+ * @param commonTypeClazz before downgrade type
+ * @param targetTypeClazz try to downgrade to type
+ * @param commonType original common type
+ * @param target target expression aka slot
+ * @param commonTypePredicate constraint for original type
+ * @param otherPredicate constraint for other expressions aka literals
+ * @param others literals
+ *
+ * @return true for should downgrade
+ */
+ private static boolean shouldDowngrade(
+ Class<? extends DataType> commonTypeClazz,
+ Class<? extends DataType> targetTypeClazz,
+ DataType commonType,
+ Expression target,
+ Function<DataType, Boolean> commonTypePredicate,
+ Function<Expression, Boolean> otherPredicate,
+ Expression... others) {
+ if (!commonTypeClazz.isInstance(commonType)) {
+ return false;
+ }
+ if (!targetTypeClazz.isInstance(target.getDataType())) {
+ return false;
+ }
+ if (!commonTypePredicate.apply(commonType)) {
+ return false;
+ }
+ for (Expression other : others) {
+ if (!otherPredicate.apply(other)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* process case when type coercion.
*/
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java
index d9a2946ba88..be465d9c371 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java
@@ -20,10 +20,17 @@ package org.apache.doris.nereids.util;
import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Divide;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.Multiply;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.Subtract;
import org.apache.doris.nereids.trees.expressions.literal.CharLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
+import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal;
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
@@ -56,6 +63,7 @@ import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.coercion.IntegralType;
+import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -735,4 +743,63 @@ public class TypeCoercionUtilsTest {
Assertions.assertEquals(expression.child(0),
new Cast(multiply.child(0), DecimalV3Type.createDecimalV3Type(10, 3)));
}
+
+ @Test
+ public void testProcessInDowngrade() {
+ // DecimalV2 slot vs DecimalV3 literal
+ InPredicate decimalDowngrade = new InPredicate(
+ new SlotReference("c1", DecimalV2Type.createDecimalV2Type(15, 6)),
+ ImmutableList.of(
+ new DecimalV3Literal(BigDecimal.valueOf(12345.1234567)),
+ new DecimalLiteral(BigDecimal.valueOf(12345.1234))));
+ decimalDowngrade = (InPredicate) TypeCoercionUtils.processInPredicate(decimalDowngrade);
+ Assertions.assertEquals(DecimalV2Type.createDecimalV2Type(16, 7), decimalDowngrade.getCompareExpr().getDataType());
+
+ // DateV1 slot vs DateV2 literal
+ InPredicate dateDowngrade = new InPredicate(
+ new SlotReference("c1", DateType.INSTANCE),
+ ImmutableList.of(
+ new DateLiteral(2024, 4, 12),
+ new DateV2Literal(2024, 4, 12)));
+ dateDowngrade = (InPredicate) TypeCoercionUtils.processInPredicate(dateDowngrade);
+ Assertions.assertEquals(DateType.INSTANCE, dateDowngrade.getCompareExpr().getDataType());
+
+ // DatetimeV1 slot vs DateLike literal
+ InPredicate datetimeDowngrade = new InPredicate(
+ new SlotReference("c1", DateTimeType.INSTANCE),
+ ImmutableList.of(
+ new DateLiteral(2024, 4, 12),
+ new DateV2Literal(2024, 4, 12),
+ new DateTimeLiteral(2024, 4, 12, 18, 25, 30),
+ new DateTimeV2Literal(2024, 4, 12, 18, 25, 30, 0)));
+ datetimeDowngrade = (InPredicate) TypeCoercionUtils.processInPredicate(datetimeDowngrade);
+ Assertions.assertEquals(DateTimeType.INSTANCE, datetimeDowngrade.getCompareExpr().getDataType());
+ }
+
+ @Test
+ public void testProcessComparisonPredicateDowngrade() {
+ // DecimalV2 slot vs DecimalV3 literal
+ EqualTo decimalDowngrade = new EqualTo(
+ new SlotReference("c1", DecimalV2Type.createDecimalV2Type(15, 6)),
+ new DecimalV3Literal(BigDecimal.valueOf(12345.1234567))
+ );
+ decimalDowngrade = (EqualTo) TypeCoercionUtils.processComparisonPredicate(decimalDowngrade);
+ Assertions.assertEquals(DecimalV2Type.createDecimalV2Type(16, 7), decimalDowngrade.left().getDataType());
+
+ // DateV1 slot vs DateV2 literal (this case cover right slot vs left literal)
+ EqualTo dateDowngrade = new EqualTo(
+ new DateV2Literal(2024, 4, 12),
+ new SlotReference("c1", DateType.INSTANCE)
+ );
+ dateDowngrade = (EqualTo) TypeCoercionUtils.processComparisonPredicate(dateDowngrade);
+ Assertions.assertEquals(DateType.INSTANCE, dateDowngrade.left().getDataType());
+
+ // DatetimeV1 slot vs DateLike literal
+ EqualTo datetimeDowngrade = new EqualTo(
+ new SlotReference("c1", DateTimeType.INSTANCE),
+ new DateTimeV2Literal(2024, 4, 12, 18, 25, 30, 0)
+ );
+ datetimeDowngrade = (EqualTo) TypeCoercionUtils.processComparisonPredicate(datetimeDowngrade);
+ Assertions.assertEquals(DateTimeType.INSTANCE, datetimeDowngrade.left().getDataType());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org