You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by en...@apache.org on 2026/05/19 02:15:40 UTC
(doris) branch master updated: [fix](fe) Avoid OFFSET path and NULL path for complex datatype when its children are accessed. (#63229)
This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 87d949800b0 [fix](fe) Avoid OFFSET path and NULL path for complex datatype when its children are accessed. (#63229)
87d949800b0 is described below
commit 87d949800b07337d3a7d128e609903db8ca4b6ff
Author: minghong <zh...@selectdb.com>
AuthorDate: Tue May 19 10:15:31 2026 +0800
[fix](fe) Avoid OFFSET path and NULL path for complex datatype when its children are accessed. (#63229)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #62205
Problem Summary: cardinality/map_size on element_at(map, key) was
collected as an OFFSET-only access path. element_at(map, key) still
needs map keys for lookup, and pushing a nested *.OFFSET predicate path
can make BE route OFFSET to an array item child and fail with an invalid
access path. Fall back to normal element access for these expressions
while preserving OFFSET-only optimization for direct array/map
cardinality.
### Release note
None
### Check List (For Author)
- Test: Unit Test
- tools/fast-compile-fe.sh
- FE_UT_PARALLEL=0 ./run-fe-ut.sh --run
org.apache.doris.nereids.rules.rewrite.PruneNestedColumnTest#testCardinalityMapElementDoesNotUseOffsetPath
- FE_UT_PARALLEL=0 ./run-fe-ut.sh --run
org.apache.doris.nereids.rules.rewrite.PruneNestedColumnTest#testStructRootMapMixedAccessKeepsKeysPath+testCardinalityMapElementDoesNotUseOffsetPath
- cd fe && mvn checkstyle:check -pl fe-core -q
- ./build.sh --fe
- Behavior changed: No
- Does this need documentation: No
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---------
Co-authored-by: Copilot <22...@users.noreply.github.com>
---
.../rewrite/AccessPathExpressionCollector.java | 19 ++
.../nereids/rules/rewrite/NestedColumnPruning.java | 294 ++++++++++++++++++---
.../rules/rewrite/PruneNestedColumnTest.java | 114 +++++++-
.../column_pruning/null_column_pruning.out | 6 +
.../string_length_column_pruning.out | 9 +
.../column_pruning/null_column_pruning.groovy | 67 ++++-
.../string_length_column_pruning.groovy | 72 ++++-
7 files changed, 523 insertions(+), 58 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
index 47d83e93ca9..680eab7796c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -45,6 +45,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySortBy;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySplit;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Cardinality;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Length;
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry;
@@ -581,6 +582,24 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
return visit(isNull, context);
}
+ @Override
+ public Void visitIf(If ifExpr, CollectorContext context) {
+ if (isFunctionNullCheckPath(context.accessPathBuilder.accessPath)) {
+ ifExpr.getCondition().accept(this, new CollectorContext(context.statementContext, context.bottomFilter));
+ ifExpr.getTrueValue().accept(this, copyContext(context));
+ ifExpr.getFalseValue().accept(this, copyContext(context));
+ return null;
+ }
+ return visit(ifExpr, context);
+ }
+
+ private static CollectorContext copyContext(CollectorContext context) {
+ CollectorContext copy = new CollectorContext(context.statementContext, context.bottomFilter);
+ copy.accessPathBuilder.addSuffix(context.accessPathBuilder.getPathList());
+ copy.type = context.type;
+ return copy;
+ }
+
@Override
public Void visitNot(Not not, CollectorContext context) {
// NOT(IS NULL) == IS NOT NULL: same null-only access pattern
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
index f9a7d4def10..c7507deb32e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -283,6 +283,7 @@ public class NestedColumnPruning implements CustomRewriter {
}
// Offset-only access (e.g. length(str_col)): type stays varchar,
// but we must still send the access path to BE so it skips the char data.
+ stripExactCoveredDataSkippingSuffixPaths(slot, allAccessPaths, allAccessPaths);
stripNullSuffixPaths(slot, allAccessPaths);
List<ColumnAccessPath> allPaths = buildColumnAccessPaths(slot, allAccessPaths);
result.put(slot.getExprId().asInt(),
@@ -348,35 +349,15 @@ public class NestedColumnPruning implements CustomRewriter {
continue;
}
+ // If a field is read in full, its metadata-only NULL/OFFSET access is redundant
+ // for any data type: e.g. [s] covers both [s.NULL] and [s.OFFSET].
+ stripExactCoveredDataSkippingSuffixPaths(slot, allAccessPaths, allAccessPaths);
+
// Strip OFFSET-suffix paths when a non-OFFSET path covers the same nested field or
// container. The overlapping array/map container may live under the root slot itself
// or under a nested struct field, so compare against the actual nested prefix instead
// of gating this logic on the root slot type.
- int slotId = slot.getExprId().asInt();
- Collection<Pair<ColumnAccessPathType, List<String>>> paths = allAccessPaths.get(slotId);
- List<List<String>> nonOffsetPaths = new ArrayList<>();
- for (Pair<ColumnAccessPathType, List<String>> p : paths) {
- List<String> path = p.second;
- if (path.isEmpty()
- || !AccessPathInfo.ACCESS_STRING_OFFSET.equals(path.get(path.size() - 1))) {
- nonOffsetPaths.add(path);
- }
- }
- List<Pair<ColumnAccessPathType, List<String>>> pathsToRemove = new ArrayList<>();
- List<Pair<ColumnAccessPathType, List<String>>> pathsToAdd = new ArrayList<>();
- for (Pair<ColumnAccessPathType, List<String>> p : new ArrayList<>(paths)) {
- OffsetPathRewrite rewrite = analyzeOffsetPathRewrite(
- slot.getDataType(), p.second, nonOffsetPaths);
- if (!rewrite.shouldRemoveOffsetPath()) {
- continue;
- }
- pathsToRemove.add(p);
- for (List<String> supplementalPath : rewrite.getSupplementalPaths()) {
- pathsToAdd.add(Pair.of(p.first, supplementalPath));
- }
- }
- paths.removeAll(pathsToRemove);
- paths.addAll(pathsToAdd);
+ stripCoveredOffsetSuffixPaths(slot, allAccessPaths, allAccessPaths);
// Strip NULL-suffix paths when a non-NULL path also exists for the same slot.
// E.g. `SELECT col FROM t WHERE col IS NULL` — full data is needed, NULL path is redundant.
@@ -399,11 +380,16 @@ public class NestedColumnPruning implements CustomRewriter {
// third: build predicate access path
for (Entry<Slot, DataTypeAccessTree> kv : slotIdToPredicateAccessTree.entrySet()) {
Slot slot = kv.getKey();
+ stripExactCoveredDataSkippingSuffixPaths(slot, predicateAccessPaths, allAccessPaths);
+ stripCoveredOffsetSuffixPaths(slot, predicateAccessPaths, allAccessPaths);
+ stripCoveredArrayNullSuffixPaths(slot, predicateAccessPaths, allAccessPaths);
stripNullSuffixPaths(slot, predicateAccessPaths);
List<ColumnAccessPath> predicatePaths =
buildColumnAccessPaths(slot, predicateAccessPaths);
AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt());
if (accessPathInfo != null) {
+ retainPredicatePathsInFinalAllAccessPaths(
+ predicatePaths, accessPathInfo.getAllAccessPaths());
accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths);
}
}
@@ -414,6 +400,8 @@ public class NestedColumnPruning implements CustomRewriter {
buildColumnAccessPaths(slot, predicateAccessPaths);
AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt());
if (accessPathInfo != null) {
+ retainPredicatePathsInFinalAllAccessPaths(
+ predicatePaths, accessPathInfo.getAllAccessPaths());
accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths);
}
}
@@ -456,6 +444,11 @@ public class NestedColumnPruning implements CustomRewriter {
return OffsetPathRewrite.keep();
}
List<String> prefix = path.subList(0, path.size() - 1);
+ return analyzePrefixCoverage(slotType, prefix, nonOffsetPaths);
+ }
+
+ private static OffsetPathRewrite analyzePrefixCoverage(
+ DataType slotType, List<String> prefix, List<List<String>> nonOffsetPaths) {
List<List<String>> supplementalPaths = new ArrayList<>();
for (List<String> nonOffset : nonOffsetPaths) {
OffsetPathRewrite candidate = compareOffsetPrefixCoverage(slotType, prefix, nonOffset);
@@ -473,6 +466,199 @@ public class NestedColumnPruning implements CustomRewriter {
return OffsetPathRewrite.rewriteWithSupplementalPaths(supplementalPaths);
}
+ /**
+ * Remove OFFSET-only paths from {@code targetAccessPaths} when data paths in
+ * {@code coveringAccessPaths} already read the same array/map/string container or a child
+ * under it.
+ *
+ * <p>Examples:
+ * <ul>
+ * <li>{@code [arr.OFFSET, arr.*.field]} becomes {@code [arr.*.field]} because the array
+ * child read must keep BE on the normal data iterator path.</li>
+ * <li>{@code [map.*.OFFSET, map.VALUES]} becomes {@code [map.KEYS, map.VALUES]} because
+ * {@code map['k']} still needs full keys for lookup, while values cover the offset.</li>
+ * </ul>
+ */
+ private static void stripCoveredOffsetSuffixPaths(
+ Slot slot, Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> targetAccessPaths,
+ Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> coveringAccessPaths) {
+ int slotId = slot.getExprId().asInt();
+ Collection<Pair<ColumnAccessPathType, List<String>>> targetPaths = targetAccessPaths.get(slotId);
+ if (targetPaths.isEmpty()) {
+ return;
+ }
+
+ List<List<String>> nonOffsetPaths = new ArrayList<>();
+ for (Pair<ColumnAccessPathType, List<String>> p : coveringAccessPaths.get(slotId)) {
+ List<String> path = p.second;
+ if (path.isEmpty()
+ || !AccessPathInfo.ACCESS_STRING_OFFSET.equals(path.get(path.size() - 1))) {
+ nonOffsetPaths.add(path);
+ }
+ }
+ for (Pair<ColumnAccessPathType, List<String>> p : targetPaths) {
+ List<String> path = p.second;
+ if (path.isEmpty()
+ || !AccessPathInfo.ACCESS_STRING_OFFSET.equals(path.get(path.size() - 1))) {
+ nonOffsetPaths.add(path);
+ }
+ }
+
+ List<Pair<ColumnAccessPathType, List<String>>> pathsToRemove = new ArrayList<>();
+ List<Pair<ColumnAccessPathType, List<String>>> pathsToAdd = new ArrayList<>();
+ for (Pair<ColumnAccessPathType, List<String>> p : new ArrayList<>(targetPaths)) {
+ OffsetPathRewrite rewrite = analyzeOffsetPathRewrite(
+ slot.getDataType(), p.second, nonOffsetPaths);
+ if (!rewrite.shouldRemoveOffsetPath()) {
+ continue;
+ }
+ pathsToRemove.add(p);
+ for (List<String> supplementalPath : rewrite.getSupplementalPaths()) {
+ pathsToAdd.add(Pair.of(p.first, supplementalPath));
+ }
+ }
+ targetPaths.removeAll(pathsToRemove);
+ targetPaths.addAll(pathsToAdd);
+ }
+
+ /**
+ * Remove array NULL-only paths from {@code targetAccessPaths} when another path already reads
+ * the same array container or data under it. This mirrors OFFSET coverage because an array
+ * element/data read must not be combined with an array NULL_MAP_ONLY read for the same prefix.
+ *
+ * <p>Examples:
+ * <ul>
+ * <li>{@code [map.VALUES.NULL, map.VALUES.*.field]} becomes
+ * {@code [map.VALUES.*.field]}.</li>
+ * <li>{@code [map.*.NULL, map.VALUES.*.field]} becomes
+ * {@code [map.KEYS, map.VALUES.*.field]} so map lookup keys are still available.</li>
+ * </ul>
+ */
+ private static void stripCoveredArrayNullSuffixPaths(
+ Slot slot, Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> targetAccessPaths,
+ Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> coveringAccessPaths) {
+ int slotId = slot.getExprId().asInt();
+ Collection<Pair<ColumnAccessPathType, List<String>>> targetPaths = targetAccessPaths.get(slotId);
+ if (targetPaths.isEmpty()) {
+ return;
+ }
+
+ List<List<String>> nonNullPaths = new ArrayList<>();
+ for (Pair<ColumnAccessPathType, List<String>> p : coveringAccessPaths.get(slotId)) {
+ List<String> path = p.second;
+ if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) {
+ nonNullPaths.add(path);
+ }
+ }
+ for (Pair<ColumnAccessPathType, List<String>> p : targetPaths) {
+ List<String> path = p.second;
+ if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) {
+ nonNullPaths.add(path);
+ }
+ }
+
+ List<Pair<ColumnAccessPathType, List<String>>> pathsToRemove = new ArrayList<>();
+ List<Pair<ColumnAccessPathType, List<String>>> pathsToAdd = new ArrayList<>();
+ for (Pair<ColumnAccessPathType, List<String>> p : new ArrayList<>(targetPaths)) {
+ List<String> path = p.second;
+ if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) {
+ continue;
+ }
+ List<String> prefix = path.subList(0, path.size() - 1);
+ Optional<DataType> prefixType = dataTypeAtPath(slot.getDataType(), prefix);
+ if (!prefixType.isPresent() || !prefixType.get().isArrayType()) {
+ continue;
+ }
+ OffsetPathRewrite rewrite = analyzePrefixCoverage(slot.getDataType(), prefix, nonNullPaths);
+ if (!rewrite.shouldRemoveOffsetPath()) {
+ continue;
+ }
+ pathsToRemove.add(p);
+ for (List<String> supplementalPath : rewrite.getSupplementalPaths()) {
+ pathsToAdd.add(Pair.of(p.first, supplementalPath));
+ }
+ }
+ targetPaths.removeAll(pathsToRemove);
+ targetPaths.addAll(pathsToAdd);
+ }
+
+ /**
+ * Remove exact metadata-only NULL/OFFSET paths when the same field is read in full.
+ * This rule is type-agnostic: once {@code s} itself is accessed, {@code s.NULL} and
+ * {@code s.OFFSET} are redundant and unsafe to keep with the full data path.
+ *
+ * <p>Examples:
+ * <ul>
+ * <li>{@code [str_col, str_col.NULL]} becomes {@code [str_col]}.</li>
+ * <li>{@code [arr, arr.OFFSET]} becomes {@code [arr]}.</li>
+ * <li>{@code [map.*, map.*.OFFSET]} becomes {@code [map.*]}.</li>
+ * </ul>
+ */
+ private static void stripExactCoveredDataSkippingSuffixPaths(
+ Slot slot, Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> targetAccessPaths,
+ Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> coveringAccessPaths) {
+ int slotId = slot.getExprId().asInt();
+ Collection<Pair<ColumnAccessPathType, List<String>>> targetPaths = targetAccessPaths.get(slotId);
+ if (targetPaths.isEmpty()) {
+ return;
+ }
+
+ List<List<String>> fullAccessPaths = new ArrayList<>();
+ for (Pair<ColumnAccessPathType, List<String>> p : coveringAccessPaths.get(slotId)) {
+ if (!isDataSkippingOnlyAccessPath(p.second)) {
+ fullAccessPaths.add(p.second);
+ }
+ }
+ for (Pair<ColumnAccessPathType, List<String>> p : targetPaths) {
+ if (!isDataSkippingOnlyAccessPath(p.second)) {
+ fullAccessPaths.add(p.second);
+ }
+ }
+
+ List<Pair<ColumnAccessPathType, List<String>>> pathsToRemove = new ArrayList<>();
+ for (Pair<ColumnAccessPathType, List<String>> p : targetPaths) {
+ List<String> path = p.second;
+ if (!isDataSkippingOnlyAccessPath(path)) {
+ continue;
+ }
+ List<String> prefix = path.subList(0, path.size() - 1);
+ for (List<String> fullAccessPath : fullAccessPaths) {
+ if (pathCoversPrefix(fullAccessPath, prefix)) {
+ pathsToRemove.add(p);
+ break;
+ }
+ }
+ }
+ targetPaths.removeAll(pathsToRemove);
+ }
+
+ private static Optional<DataType> dataTypeAtPath(DataType slotType, List<String> path) {
+ if (path.isEmpty()) {
+ return Optional.empty();
+ }
+ DataType currentType = slotType;
+ for (int i = 1; i < path.size(); i++) {
+ String component = path.get(i);
+ if (currentType.isStructType()) {
+ StructField field = ((StructType) currentType).getField(component);
+ if (field == null) {
+ return Optional.empty();
+ }
+ currentType = field.getDataType();
+ } else if (currentType.isArrayType()) {
+ if (!AccessPathInfo.ACCESS_ALL.equals(component)) {
+ return Optional.empty();
+ }
+ currentType = ((ArrayType) currentType).getItemType();
+ } else if (currentType.isMapType()) {
+ currentType = descendMapType((MapType) currentType, component);
+ } else {
+ return Optional.empty();
+ }
+ }
+ return Optional.of(currentType);
+ }
+
private static OffsetPathRewrite compareOffsetPrefixCoverage(
DataType slotType, List<String> prefix, List<String> nonOffset) {
if (nonOffset.isEmpty()) {
@@ -590,21 +776,27 @@ public class NestedColumnPruning implements CustomRewriter {
}
/**
- * Strip NULL-suffix paths that are redundant because a non-NULL path reads the same
- * column/subcolumn in full (its data inherently includes the null flag).
+ * Strip NULL-suffix paths that are redundant because a non-NULL path reads child
+ * data below the same prefix or reads an OFFSET path over the same prefix.
*
- * For example, [int_col, NULL] is removed when [int_col] exists — reading the full
- * column includes its null flag.
+ * <p>Examples:
+ * <ul>
+ * <li>{@code [struct_col.NULL, struct_col.city]} becomes {@code [struct_col.city]}.</li>
+ * <li>{@code [str_col.NULL, str_col.OFFSET]} becomes {@code [str_col.OFFSET]} because
+ * the offset read can provide nullness for variable-length columns.</li>
+ * </ul>
*
- * A parent NULL path must also be removed when any child path is required under the
+ * <p>A parent NULL path must also be removed when any child path is required under the
* same prefix, e.g. [struct_col, NULL] with [struct_col, city]. This looks like the
- * parent null map is still useful for predicates, but it cannot be kept in
+ * parent null map may still be useful for predicates, but it cannot be kept in
* allAccessPaths with the current BE iterator contract: Struct/Array/Map iterators
* treat a leading NULL sub-path as NULL_MAP_ONLY and skip all children. If FE kept
* [struct_col.NULL, struct_col.city] in allAccessPaths, BE would read only the
* struct null map and default-fill city instead of routing the city child iterator.
- * predicateAccessPaths still retains the NULL path, while the normal nullable
- * container read materializes the parent null map together with required children.
+ * When the NULL path is removed from allAccessPaths, it must also be removed from
+ * predicateAccessPaths so the BE can rely on predicate paths being a subset of all
+ * paths. The normal nullable container read materializes the parent null map
+ * together with required children.
*/
private static void stripNullSuffixPaths(
Slot slot, Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> allAccessPaths) {
@@ -655,10 +847,43 @@ public class NestedColumnPruning implements CustomRewriter {
}
}
+ /**
+ * Keep predicate access paths as a subset of final all access paths after NULL/OFFSET cleanup.
+ * Predicate paths are built from filter expressions first, but later all-path rewrites may drop
+ * metadata-only paths or collapse paths to whole-column access. Any predicate path not present
+ * in final all paths must be removed before sending access info to BE.
+ *
+ * <p>Examples:
+ * <ul>
+ * <li>All paths {@code [s]}, predicate paths {@code [s.city.NULL]} becomes no predicate
+ * paths after parent NULL removal.</li>
+ * <li>All paths {@code [s.city.NULL, s.zip]}, predicate paths
+ * {@code [s.NULL, s.city.NULL]} becomes {@code [s.city.NULL]}.</li>
+ * </ul>
+ */
+ private static void retainPredicatePathsInFinalAllAccessPaths(
+ List<ColumnAccessPath> predicatePaths, List<ColumnAccessPath> allPaths) {
+ if (predicatePaths.isEmpty()) {
+ return;
+ }
+
+ List<ColumnAccessPath> toRemove = new ArrayList<>();
+ for (ColumnAccessPath predicatePath : predicatePaths) {
+ if (!allPaths.contains(predicatePath)) {
+ toRemove.add(predicatePath);
+ }
+ }
+ predicatePaths.removeAll(toRemove);
+ }
+
private static boolean hasStrictPrefix(List<String> path, List<String> prefix) {
return path.size() > prefix.size() && path.subList(0, prefix.size()).equals(prefix);
}
+ private static boolean pathCoversPrefix(List<String> path, List<String> prefix) {
+ return prefix.size() >= path.size() && prefix.subList(0, path.size()).equals(path);
+ }
+
private static List<ColumnAccessPath> buildColumnAccessPaths(
Slot slot, Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> accessPaths) {
List<ColumnAccessPath> paths = new ArrayList<>();
@@ -682,7 +907,8 @@ public class NestedColumnPruning implements CustomRewriter {
if (accessWholeColumn) {
SlotReference slotReference = (SlotReference) slot;
String wholeColumnName = slotReference.getOriginalColumn().get().getName();
- return ImmutableList.of(new ColumnAccessPath(accessWholeColumnType, ImmutableList.of(wholeColumnName)));
+ return new ArrayList<>(
+ ImmutableList.of(new ColumnAccessPath(accessWholeColumnType, ImmutableList.of(wholeColumnName))));
}
return paths;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 8d8d3441fa6..4c200629ac0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -117,6 +117,23 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
+ " >\n"
+ ") properties ('replication_num'='1')");
+ createTable("create table nested_array_tbl(\n"
+ + " id int,\n"
+ + " a array<array<int>>\n"
+ + ") properties ('replication_num'='1')");
+
+ createTable("create table map_array_tbl(\n"
+ + " id int,\n"
+ + " map_arr_col map<string, array<int>>\n"
+ + ") properties ('replication_num'='1')");
+
+ createTable("create table map_array_value_tbl(\n"
+ + " id int,\n"
+ + " s struct<\n"
+ + " m: map<string, array<struct<verified: boolean, value: int>>>\n"
+ + " >\n"
+ + ") properties ('replication_num'='1')");
+
connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name());
connectContext.getSessionVariable().enableNereidsTimeout = false;
}
@@ -174,6 +191,80 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
ImmutableList.of(path("s", "m", "*", "OFFSET"), path("s", "m", "VALUES", "OFFSET")));
}
+ @Test
+ public void testCardinalityArrayElementKeepsOffsetPath() throws Exception {
+ assertAllAccessPathsContain(
+ "select cardinality(element_at(a, 1)) from nested_array_tbl",
+ ImmutableList.of(path("a", "*", "OFFSET")),
+ ImmutableList.of(path("a", "*")));
+ }
+
+ @Test
+ public void testCardinalityMapElementKeepsValueOffsetPath() throws Exception {
+ assertColumn("select cardinality(map_arr_col['a']) from map_array_tbl",
+ "map<text,array<int>>",
+ ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES", "OFFSET")),
+ ImmutableList.of());
+ }
+
+ @Test
+ public void testFullFieldAccessStripsExactDataSkippingPath() throws Exception {
+ assertColumn("select struct_element(s, 'city') from tbl "
+ + "where struct_element(s, 'city') is null",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of());
+
+ assertColumn("select cardinality(struct_element(s, 'data')), struct_element(s, 'data') from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data")),
+ ImmutableList.of());
+
+ assertColumn("select cardinality(a), a from nested_array_tbl",
+ "array<array<int>>",
+ ImmutableList.of(path("a")),
+ ImmutableList.of());
+
+ assertColumn("select cardinality(map_arr_col['a']), map_arr_col['a'] from map_array_tbl",
+ "map<text,array<int>>",
+ ImmutableList.of(path("map_arr_col", "*")),
+ ImmutableList.of());
+ }
+
+ @Test
+ public void testCardinalityMapElementOffsetCoveredByValueFieldAccess() throws Exception {
+ Pair<PhysicalPlan, List<SlotDescriptor>> result = collectComplexSlots(
+ "select struct_element(element_at(element_at(struct_element(s, 'm'), 'null'), 1), 'verified') "
+ + "from map_array_value_tbl "
+ + "where cardinality(element_at(struct_element(s, 'm'), 'null')) > 0");
+ TreeSet<ColumnAccessPath> allAccessPaths = new TreeSet<>();
+ TreeSet<ColumnAccessPath> predicateAccessPaths = new TreeSet<>();
+ for (SlotDescriptor slotDescriptor : result.second) {
+ allAccessPaths.addAll(slotDescriptor.getAllAccessPaths());
+ predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths());
+ }
+ Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "*", "*", "verified")));
+ Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "OFFSET")));
+ Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "*", "OFFSET")));
+ }
+
+ @Test
+ public void testMapElementArrayNullPathCoveredByValueFieldAccess() throws Exception {
+ Pair<PhysicalPlan, List<SlotDescriptor>> result = collectComplexSlots(
+ "select struct_element(element_at(element_at(struct_element(s, 'm'), 'null'), 1), 'verified') "
+ + "from map_array_value_tbl "
+ + "where element_at(struct_element(s, 'm'), 'null') is null");
+ TreeSet<ColumnAccessPath> allAccessPaths = new TreeSet<>();
+ TreeSet<ColumnAccessPath> predicateAccessPaths = new TreeSet<>();
+ for (SlotDescriptor slotDescriptor : result.second) {
+ allAccessPaths.addAll(slotDescriptor.getAllAccessPaths());
+ predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths());
+ }
+ Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "*", "*", "verified")));
+ Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "NULL")));
+ Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "*", "NULL")));
+ }
+
@Test
public void testVariantAccessPath() throws Exception {
assertColumn("select v['a']['B'] from variant_tbl",
@@ -554,12 +645,12 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
assertColumn("select map_keys(map_col) from str_tbl where map_keys(map_col) is null",
"map<text,text>",
ImmutableList.of(path("map_col", "KEYS")),
- ImmutableList.of(path("map_col", "NULL"))
+ ImmutableList.of()
);
assertColumn("select map_values(map_col) from str_tbl where map_values(map_col) is null",
"map<text,text>",
ImmutableList.of(path("map_col", "VALUES")),
- ImmutableList.of(path("map_col", "NULL"))
+ ImmutableList.of()
);
}
@@ -568,7 +659,12 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
assertColumn("select s from tbl where struct_element(s, 'city') is not null",
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
ImmutableList.of(path("s")),
- ImmutableList.of(path("s", "city", "NULL"))
+ ImmutableList.of()
+ );
+ assertColumn("select s from tbl where struct_element(s, 'city') is null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
);
assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null",
@@ -580,7 +676,7 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null and struct_element(s, 'data') is not null",
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
ImmutableList.of(path("s", "city", "NULL"), path("s", "data")),
- ImmutableList.of(path("s", "city", "NULL"), path("s", "data", "NULL"))
+ ImmutableList.of(path("s", "city", "NULL"))
);
}
@@ -1204,6 +1300,7 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
TreeSet<ColumnAccessPath> actualPredicateAccessPaths
= new TreeSet<>(slotDescriptor.getPredicateAccessPaths());
Assertions.assertEquals(expectPredicateAccessPathSet, actualPredicateAccessPaths);
+ Assertions.assertTrue(actualAllAccessPaths.containsAll(actualPredicateAccessPaths));
Map<Integer, DataType> slotIdToDataTypes = new LinkedHashMap<>();
Consumer<Expression> assertHasSameType = e -> {
@@ -1278,21 +1375,22 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
// Parent NULL path must be stripped from allPaths when a child path is also required.
// Otherwise BE StructFileColumnIterator sees the parent NULL sub-path first, switches
// the whole struct iterator to NULL_MAP_ONLY, and skips the child iterator.
- // predicateAccessPaths keeps [s, NULL] because the predicate itself still uses it.
+ // predicateAccessPaths drops [s, NULL] too, keeping it a subset of allAccessPaths.
assertColumn("select struct_element(s, 'city') from tbl where s is null",
"struct<city:text>",
ImmutableList.of(path("s", "city")),
- ImmutableList.of(path("s", "NULL")));
+ ImmutableList.of());
// This shape is closer to the production bug: one predicate needs the parent
// null map, another predicate needs a child null map, and the projection needs
// a different child data path. The parent [s.NULL] cannot remain in allPaths
- // with [s.data], but both predicate NULL paths must remain in predicate paths.
+ // with [s.data], so it is also removed from predicate paths; [s.city.NULL] stays
+ // because it is still present in allPaths.
assertColumn("select struct_element(s, 'data') from tbl "
+ "where s is null or struct_element(s, 'city') is null",
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
ImmutableList.of(path("s", "city", "NULL"), path("s", "data")),
- ImmutableList.of(path("s", "NULL"), path("s", "city", "NULL")));
+ ImmutableList.of(path("s", "city", "NULL")));
}
@Test
diff --git a/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out b/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out
index 440b393001c..5e18f989b1a 100644
--- a/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out
+++ b/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out
@@ -3,6 +3,8 @@
-- !2 --
+-- !string_full_access_strips_null --
+
-- !3 --
1
@@ -14,8 +16,12 @@
-- !6 --
+-- !array_full_access_strips_null --
+
-- !7 --
+-- !map_full_access_strips_null --
+
-- !8 --
-- !9 --
diff --git a/regression-test/data/nereids_rules_p0/column_pruning/string_length_column_pruning.out b/regression-test/data/nereids_rules_p0/column_pruning/string_length_column_pruning.out
index 658e8ebbee8..36b77d17167 100644
--- a/regression-test/data/nereids_rules_p0/column_pruning/string_length_column_pruning.out
+++ b/regression-test/data/nereids_rules_p0/column_pruning/string_length_column_pruning.out
@@ -1,4 +1,13 @@
-- This file is automatically generated. You should know what you did if you want to edit this
+-- !array_full_access_strips_offset --
+1 3 [1, 2, 3]
+
+-- !map_element_full_access_strips_offset --
+1 2 [1, 2]
+
+-- !map_value_array_predicate_offset_covered --
+true
+
-- !arr_struct_mixed --
1 true 10
2 true 30
diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy
index 2707670f7d8..cbbeac29357 100644
--- a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy
+++ b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy
@@ -24,8 +24,8 @@
// nested columns: <col>: all access paths: [<col>.NULL]
//
// When the same column is also accessed for data (e.g., projected or used in
-// struct_element), the NULL-only path must be stripped from allAccessPaths but
-// preserved in predicateAccessPaths.
+// struct_element), the NULL-only path must be stripped from allAccessPaths and
+// predicateAccessPaths unless the same path is still present in allAccessPaths.
suite("null_column_pruning") {
sql """ DROP TABLE IF EXISTS ncp_tbl """
@@ -69,6 +69,19 @@ suite("null_column_pruning") {
order_qt_2 "select 1 from ncp_tbl where str_col is null";
+ // Direct full access to the same field covers its null flag for any data type.
+ // The exact [str_col.NULL] metadata path must be removed.
+ explain {
+ sql "select id, str_col from ncp_tbl where str_col is null"
+ notContains "str_col.NULL"
+ notContains "predicate access paths:"
+ }
+
+ order_qt_string_full_access_strips_null """
+ select id, str_col from ncp_tbl where str_col is null
+ order by id
+ """
+
// ─── String IS NOT NULL only ────────────────────────────────────────────────
explain {
sql "select 1 from ncp_tbl where str_col is not null"
@@ -106,6 +119,19 @@ suite("null_column_pruning") {
order_qt_6 "select 1 from ncp_tbl where arr_col is null";
+ explain {
+ sql "select id, arr_col from ncp_tbl where arr_col is null"
+ contains "nested columns"
+ contains "all access paths: [arr_col]"
+ notContains "arr_col.NULL"
+ notContains "predicate access paths:"
+ }
+
+ order_qt_array_full_access_strips_null """
+ select id, arr_col from ncp_tbl where arr_col is null
+ order by id
+ """
+
// ─── Map IS NULL only ───────────────────────────────────────────────────────
explain {
sql "select 1 from ncp_tbl where map_col is null"
@@ -115,6 +141,19 @@ suite("null_column_pruning") {
order_qt_7 "select 1 from ncp_tbl where map_col is null";
+ explain {
+ sql "select id, map_col from ncp_tbl where map_col is null"
+ contains "nested columns"
+ contains "all access paths: [map_col]"
+ notContains "map_col.NULL"
+ notContains "predicate access paths:"
+ }
+
+ order_qt_map_full_access_strips_null """
+ select id, map_col from ncp_tbl where map_col is null
+ order by id
+ """
+
// ─── Int IS NULL only ───────────────────────────────────────────────────────
// Nullable primitive type (INT) accessed only via IS NULL → emit [int_col, NULL]
// access path so BE only reads the null flag.
@@ -143,7 +182,7 @@ suite("null_column_pruning") {
sql "select int_col from ncp_tbl where int_col is null"
contains "nested columns"
contains "all access paths: [int_col]"
- contains "predicate access paths: [int_col.NULL]"
+ notContains "predicate access paths:"
}
order_qt_10 "select int_col from ncp_tbl where int_col is null";
@@ -153,14 +192,14 @@ suite("null_column_pruning") {
// The parent struct_col.NULL path must NOT stay in allAccessPaths with child paths.
// BE StructFileColumnIterator treats a leading NULL sub-path as NULL_MAP_ONLY; if
// allAccessPaths were [struct_col.NULL, struct_col.city], BE would skip the city
- // child iterator and default-fill the projected value. predicateAccessPaths still
- // keeps struct_col.NULL so the predicate requirement is visible, while the normal
- // nullable struct read materializes the parent null map together with child data.
+ // child iterator and default-fill the projected value. The normal nullable struct
+ // read materializes the parent null map together with child data, and
+ // predicateAccessPaths is filtered so it remains a subset of allAccessPaths.
explain {
sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null"
contains "nested columns"
contains "all access paths: [struct_col.city]"
- contains "predicate access paths: [struct_col.NULL]"
+ notContains "predicate access paths:"
}
order_qt_11 "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null";
@@ -174,7 +213,7 @@ suite("null_column_pruning") {
sql "select struct_element(struct_col, 'zip') from ncp_tbl where struct_col is null or struct_element(struct_col, 'city') is null"
contains "nested columns"
contains "all access paths: [struct_col.city.NULL, struct_col.zip]"
- contains "predicate access paths: [struct_col.NULL, struct_col.city.NULL]"
+ contains "predicate access paths: [struct_col.city.NULL]"
}
order_qt_parent_null_with_child_data "select struct_element(struct_col, 'zip') from ncp_tbl where struct_col is null or struct_element(struct_col, 'city') is null";
@@ -186,7 +225,7 @@ suite("null_column_pruning") {
sql "select struct_col from ncp_tbl where struct_col is null"
contains "nested columns"
contains "all access paths: [struct_col]"
- contains "predicate access paths: [struct_col.NULL]"
+ notContains "predicate access paths:"
}
order_qt_12 "select struct_col from ncp_tbl where struct_col is null";
@@ -200,7 +239,7 @@ suite("null_column_pruning") {
sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_element(struct_col, 'city') is null"
contains "nested columns"
contains "all access paths: [struct_col.city]"
- contains "predicate access paths: [struct_col.city.NULL]"
+ notContains "predicate access paths:"
}
order_qt_13 "select struct_element(struct_col, 'city') from ncp_tbl where struct_element(struct_col, 'city') is null";
@@ -355,13 +394,13 @@ suite("null_column_pruning") {
// ─── Mixed: map_keys IS NULL + map_keys projected ──────────────────────────
// Projection needs key data, while the predicate checks whether the parent map
- // is NULL. The parent NULL path is kept only in predicateAccessPaths so BE does
- // not switch the whole map iterator to NULL_MAP_ONLY and skip the keys child.
+ // is NULL. The parent NULL path must not stay in either access path list, so BE
+ // does not switch the whole map iterator to NULL_MAP_ONLY and skip the keys child.
explain {
sql "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null"
contains "nested columns"
contains "all access paths: [map_col.KEYS]"
- contains "predicate access paths: [map_col.NULL]"
+ notContains "predicate access paths:"
}
order_qt_25 "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null";
@@ -373,7 +412,7 @@ suite("null_column_pruning") {
sql "select map_values(map_col) from ncp_tbl where map_values(map_col) is null"
contains "nested columns"
contains "all access paths: [map_col.VALUES]"
- contains "predicate access paths: [map_col.NULL]"
+ notContains "predicate access paths:"
}
order_qt_26 "select map_values(map_col) from ncp_tbl where map_values(map_col) is null";
diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy
index a0da286867c..2a4d56b570b 100644
--- a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy
+++ b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy
@@ -45,7 +45,8 @@ suite("string_length_column_pruning") {
struct_col STRUCT<f1: INT, f3: STRING>,
arr_col ARRAY<INT>,
map_col MAP<STRING, STRING>,
- map_arr_col MAP<STRING, ARRAY<INT>>
+ map_arr_col MAP<STRING, ARRAY<INT>>,
+ map_arr_struct_col MAP<STRING, ARRAY<STRUCT<verified: BOOLEAN, value: INT>>>
) ENGINE = OLAP
DUPLICATE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 1
@@ -53,7 +54,9 @@ suite("string_length_column_pruning") {
"""
sql """
INSERT INTO slcp_str_tbl VALUES
- (1, 'hello', named_struct('f1', 10, 'f3', 'world'), [1, 2, 3], {'a': 'x', 'b': 'y'}, {'a': [1, 2], 'b': [3]})
+ (1, 'hello', named_struct('f1', 10, 'f3', 'world'), [1, 2, 3], {'a': 'x', 'b': 'y'},
+ {'a': [1, 2], 'b': [3]},
+ map('a', array(named_struct('verified', true, 'value', 10))))
"""
// ─── Optimizable cases ──────────────────────────────────────────────────────
@@ -151,6 +154,21 @@ suite("string_length_column_pruning") {
notContains "type=bigint"
}
+ // Full access to the same array field covers its OFFSET metadata for any data type.
+ explain {
+ sql "select id, cardinality(arr_col), arr_col from slcp_str_tbl"
+ contains "nested columns"
+ contains "all access paths: [arr_col]"
+ notContains "arr_col.OFFSET"
+ notContains "predicate access paths:"
+ notContains "type=bigint"
+ }
+
+ order_qt_array_full_access_strips_offset """
+ select id, cardinality(arr_col), arr_col from slcp_str_tbl
+ order by id
+ """
+
// ─── Map column cases ────────────────────────────────────────────────────────
// cardinality(map_col): only the offset array is needed → OFFSET access path emitted,
@@ -254,6 +272,56 @@ suite("string_length_column_pruning") {
notContains "type=bigint"
}
+ // value array item also accessed directly → full VALUES item path covers value OFFSET.
+ explain {
+ sql "select cardinality(map_arr_struct_col['a']), map_arr_struct_col['a'][1].verified from slcp_str_tbl"
+ contains "nested columns"
+ contains "map_arr_struct_col.*.*.verified"
+ notContains "map_arr_struct_col.*.OFFSET"
+ notContains "type=bigint"
+ }
+
+ explain {
+ sql "select id, cardinality(map_arr_col['a']), map_arr_col['a'] from slcp_str_tbl"
+ contains "nested columns"
+ contains "all access paths: [map_arr_col.*]"
+ notContains "map_arr_col.*.OFFSET"
+ notContains "predicate access paths:"
+ notContains "type=bigint"
+ }
+
+ order_qt_map_element_full_access_strips_offset """
+ select id, cardinality(map_arr_col['a']), map_arr_col['a'] from slcp_str_tbl
+ order by id
+ """
+
+ // Predicate OFFSET path must also be removed when the projected value field already
+ // makes the corresponding array data path available. predicateAccessPaths remains a
+ // subset of allAccessPaths.
+ explain {
+ sql "select map_arr_struct_col['a'][1].verified from slcp_str_tbl where cardinality(map_arr_struct_col['a']) > 0"
+ contains "nested columns"
+ contains "all access paths: [map_arr_struct_col.*.*.verified]"
+ notContains "map_arr_struct_col.*.OFFSET"
+ notContains "predicate access paths:"
+ notContains "type=bigint"
+ }
+
+ order_qt_map_value_array_predicate_offset_covered """
+ select map_arr_struct_col['a'][1].verified from slcp_str_tbl
+ where cardinality(map_arr_struct_col['a']) > 0
+ order by 1
+ """
+
+ // value array item also accessed directly → full VALUES item path covers value NULL.
+ explain {
+ sql "select map_arr_struct_col['a'][1].verified from slcp_str_tbl where map_arr_struct_col['a'] is null"
+ contains "nested columns"
+ contains "map_arr_struct_col.*.*.verified"
+ notContains "map_arr_struct_col.*.NULL"
+ notContains "predicate access paths:"
+ }
+
// ─── Non-optimizable cases ──────────────────────────────────────────────────
// str_col also projected directly → full chars data needed, OFFSET path suppressed.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org