You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2025/12/23 06:54:31 UTC
(doris) 03/04: [fix](nereids) fix prune map type cause backend core (#58573)
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch cherry-pick-nested_column_prune_4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f61d01c92deba8b4cfa9b570e49ba0dd71ce64c3
Author: 924060929 <la...@selectdb.com>
AuthorDate: Tue Dec 2 14:12:56 2025 +0800
[fix](nereids) fix prune map type cause backend core (#58573)
fix prune map type cause backend core, when the map type is changed, we
should not prune the nested column type, introduced by #57204
(cherry picked from commit 1d7f6c422fe20dfa8c64ae9a36ee093058756f04)
---
.../rewrite/AccessPathExpressionCollector.java | 48 +++++++++++++++++++++-
.../rules/rewrite/PruneNestedColumnTest.java | 9 +++-
2 files changed, 55 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
index 9c665f379e9..a46bc67a283 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -50,7 +50,9 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.MapValues;
import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
+import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.MapType;
import org.apache.doris.nereids.types.NestedColumnPrunable;
import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
@@ -65,6 +67,7 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
import java.util.Stack;
@@ -135,7 +138,8 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
public Void visitCast(Cast cast, CollectorContext context) {
if (!context.accessPathBuilder.isEmpty()
&& cast.getDataType() instanceof NestedColumnPrunable
- && cast.child().getDataType() instanceof NestedColumnPrunable) {
+ && cast.child().getDataType() instanceof NestedColumnPrunable
+ && !mapTypeIsChanged(cast.child().getDataType(), cast.getDataType(), false)) {
DataTypeAccessTree castTree = DataTypeAccessTree.of(cast.getDataType(), TAccessPathType.DATA);
DataTypeAccessTree originTree = DataTypeAccessTree.of(cast.child().getDataType(), TAccessPathType.DATA);
@@ -509,4 +513,46 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
return path.hashCode();
}
}
+
+ // if the map type is changed, we can not prune the type, because the map type need distinct the keys,
+ // e.g. select map_values(cast(map(3.0, 1, 3.1, 2) as map<int, int>));
+ // the result is [2] because the keys: 3.0 and 3.1 will cast to 3 and the second entry remained.
+ // backend will throw exception because it can not only access the values without the cast keys,
+ // so we should check whether the map type is changed, if not changed, we can prune the type.
+ private static boolean mapTypeIsChanged(DataType originType, DataType castType, boolean inMap) {
+ if (originType.isMapType()) {
+ MapType originMapType = (MapType) originType;
+ MapType castMapType = (MapType) castType;
+ if (mapTypeIsChanged(originMapType.getKeyType(), castMapType.getKeyType(), true)
+ || mapTypeIsChanged(originMapType.getValueType(), castMapType.getValueType(), true)) {
+ return true;
+ }
+ return false;
+ } else if (originType.isStructType()) {
+ StructType originStructType = (StructType) originType;
+ StructType castStructType = (StructType) castType;
+ List<Entry<String, StructField>> originFields
+ = new ArrayList<>(originStructType.getNameToFields().entrySet());
+ List<Entry<String, StructField>> castFields
+ = new ArrayList<>(castStructType.getNameToFields().entrySet());
+
+ for (int i = 0; i < originFields.size(); i++) {
+ DataType originFieldType = originFields.get(i).getValue().getDataType();
+ DataType castFieldType = castFields.get(i).getValue().getDataType();
+ if (mapTypeIsChanged(originFieldType, castFieldType, inMap)) {
+ return true;
+ }
+ }
+ return false;
+ } else if (originType.isArrayType()) {
+ ArrayType originArrayType = (ArrayType) originType;
+ ArrayType castArrayType = (ArrayType) castType;
+ return mapTypeIsChanged(originArrayType.getItemType(), castArrayType.getItemType(), inMap);
+ } else if (inMap) {
+ return !originType.equals(castType);
+ } else {
+ // other type changed which not in map will not affect the map
+ return false;
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 3d22e1e6c28..2649b29cdff 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -140,13 +140,20 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
@Test
public void testPruneCast() throws Exception {
+ // the map type is changed, so we can not prune type
+ assertColumn("select struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:int,y:int>>>>), 'k') from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+
assertColumn("select struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:int,y:double>>>>), 'k') from tbl",
"struct<city:text>",
ImmutableList.of(path("s", "city")),
ImmutableList.of()
);
- assertColumn("select struct_element(map_values(struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:double,y:double>>>>), 'l')[0])[0], 'x') from tbl",
+ assertColumn("select struct_element(map_values(struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:int,y:double>>>>), 'l')[0])[0], 'x') from tbl",
"struct<data:array<map<int,struct<a:int>>>>",
ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
ImmutableList.of()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org