You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by en...@apache.org on 2026/05/13 07:20:29 UTC
(doris) branch master updated: [opt](nereids) Optimize I/O operations for the IS NULL predicate (#62304)
This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 85ede0bafba [opt](nereids) Optimize I/O operations for the IS NULL predicate (#62304)
85ede0bafba is described below
commit 85ede0bafba0167f92960c46bd1858b306416b43
Author: minghong <zh...@selectdb.com>
AuthorDate: Wed May 13 15:20:19 2026 +0800
[opt](nereids) Optimize I/O operations for the IS NULL predicate (#62304)
### What problem does this PR solve?
Treat nullable fields as a combination of a nullable flag and data. When
evaluating the `col IS NULL` predicate, use the NestedColumnPruning rule
to prune the col field to col.NULL, thereby saving I/O on the data.
---
be/src/storage/segment/column_reader.cpp | 2 -
.../org/apache/doris/analysis/SlotDescriptor.java | 4 +
.../org/apache/doris/nereids/CascadesContext.java | 9 +
.../mv/AbstractMaterializedViewRule.java | 2 +-
.../exploration/mv/MaterializedViewUtils.java | 11 +
.../nereids/rules/exploration/mv/StructInfo.java | 2 +-
.../rewrite/AccessPathExpressionCollector.java | 99 ++++-
.../nereids/rules/rewrite/AccessPathInfo.java | 3 +
.../rules/rewrite/AccessPathPlanCollector.java | 37 +-
.../nereids/rules/rewrite/NestedColumnPruning.java | 248 ++++++++++-
.../nereids/rules/rewrite/SlotTypeReplacer.java | 3 +-
.../rules/rewrite/PruneNestedColumnTest.java | 412 ++++++++++++-----
.../column_pruning/null_column_pruning.out | 101 +++++
.../mv/unsafe_equals/null_un_safe_equals.out | 12 +
.../column_pruning/null_column_pruning.groovy | 493 +++++++++++++++++++++
.../string_length_column_pruning.groovy | 17 +
.../mv/unsafe_equals/null_un_safe_equals.groovy | 17 +-
17 files changed, 1326 insertions(+), 146 deletions(-)
diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp
index 189846ad339..2e99eb259b0 100644
--- a/be/src/storage/segment/column_reader.cpp
+++ b/be/src/storage/segment/column_reader.cpp
@@ -2237,8 +2237,6 @@ Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t co
const size_t base_size = null_map_data.size();
null_map_data.resize(base_size + count);
- nullable_col.get_nested_column().insert_many_defaults(count);
-
size_t remaining = count;
size_t total_read_count = 0;
size_t nrows_to_read = 0;
diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
index ec8d98c4fbb..ff00fb5c17a 100644
--- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
+++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
@@ -65,6 +65,10 @@ public class SlotDescriptor {
this.id = id;
this.parentId = parentId;
this.isNullable = true;
+ this.allAccessPaths = Collections.emptyList();
+ this.predicateAccessPaths = Collections.emptyList();
+ this.displayAllAccessPaths = Collections.emptyList();
+ this.displayPredicateAccessPaths = Collections.emptyList();
}
public SlotId getId() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index e46d2729899..e1036af9208 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -107,6 +107,7 @@ public class CascadesContext implements ScheduleContext {
private Optional<Scope> outerScope = Optional.empty();
private boolean isRewriteRoot;
+ private boolean isMaterializedViewRewritePlanFragment;
private volatile boolean isTimeout = false;
// current process subtree, represent outer plan if empty
@@ -402,6 +403,14 @@ public class CascadesContext implements ScheduleContext {
return isRewriteRoot;
}
+ public void setMaterializedViewRewritePlanFragment(boolean materializedViewRewritePlanFragment) {
+ isMaterializedViewRewritePlanFragment = materializedViewRewritePlanFragment;
+ }
+
+ public boolean isMaterializedViewRewritePlanFragment() {
+ return isMaterializedViewRewritePlanFragment;
+ }
+
public Optional<Scope> getOuterScope() {
return outerScope;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 64361ba9a97..a293e94d775 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -312,7 +312,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
childContext -> {
Rewriter.getWholeTreeRewriter(childContext).execute();
return childContext.getRewritePlan();
- }, rewrittenPlan, queryPlan, false);
+ }, rewrittenPlan, queryPlan, false, true);
if (rewrittenPlan == null) {
continue;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 4c3194703f5..009671e9ef3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -343,6 +343,16 @@ public class MaterializedViewUtils {
public static Plan rewriteByRules(
CascadesContext cascadesContext, Function<CascadesContext, Plan> planRewriter,
Plan rewrittenPlan, Plan originPlan, boolean mvRewrite) {
+ return rewriteByRules(cascadesContext, planRewriter, rewrittenPlan, originPlan, mvRewrite, false);
+ }
+
+ /**
+ * Optimize by rules, this support optimize by custom rules by define different rewriter according to different
+ * rules, this method is only for materialized view rewrite
+ */
+ public static Plan rewriteByRules(
+ CascadesContext cascadesContext, Function<CascadesContext, Plan> planRewriter,
+ Plan rewrittenPlan, Plan originPlan, boolean mvRewrite, boolean materializedViewRewritePlanFragment) {
if (originPlan == null || rewrittenPlan == null) {
return null;
}
@@ -358,6 +368,7 @@ public class MaterializedViewUtils {
CascadesContext rewrittenPlanContext = CascadesContext.initContext(
cascadesContext.getStatementContext(), rewrittenPlan,
cascadesContext.getCurrentJobContext().getRequiredProperties());
+ rewrittenPlanContext.setMaterializedViewRewritePlanFragment(materializedViewRewritePlanFragment);
// Tmp old disable rule variable
Set<String> oldDisableRuleNames = rewrittenPlanContext.getStatementContext().getConnectContext()
.getSessionVariable()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index a45863aa32c..d4e0e3f5003 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -981,7 +981,7 @@ public class StructInfo {
return Pair.of(MaterializedViewUtils.rewriteByRules(parentCascadesContext, context -> {
Rewriter.getWholeTreeRewriter(context).execute();
return context.getRewritePlan();
- }, queryPlanWithUnionFilter, queryPlan, false), true);
+ }, queryPlanWithUnionFilter, queryPlan, false, true), true);
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
index 4b2493f817e..47d83e93ca9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -26,6 +26,8 @@ import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayCount;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
@@ -118,7 +120,14 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
if (slotReference.hasSubColPath()) {
path.addAll(slotReference.getSubPath());
}
- path.addAll(context.accessPathBuilder.getPathList());
+ // Strip NULL suffix for variant sub-column access — null-flag-only optimization
+ // does not apply to variant sub-column data layout.
+ List<String> builderPath = context.accessPathBuilder.getPathList();
+ if (builderPath.size() > 1
+ && AccessPathInfo.ACCESS_NULL.equals(builderPath.get(builderPath.size() - 1))) {
+ builderPath = new ArrayList<>(builderPath.subList(0, builderPath.size() - 1));
+ }
+ path.addAll(builderPath);
int slotId = slotReference.getExprId().asInt();
slotToAccessPaths.put(slotId, new CollectAccessPathResult(
path, context.bottomFilter, ColumnAccessPathType.DATA));
@@ -133,8 +142,8 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
if (dataType.isStringLikeType()) {
int slotId = slotReference.getExprId().asInt();
if (!context.accessPathBuilder.isEmpty()) {
- // Accessed via an offset-only function (e.g. length()).
- // Builder already has "offset" at the tail; add the column name as prefix.
+ // Accessed via an offset-only function (e.g. length()) or null-check (IS NULL).
+ // Builder already has "OFFSET"/"NULL" at the tail; add the column name as prefix.
context.accessPathBuilder.addPrefix(slotReference.getName());
ImmutableList<String> path = ImmutableList.copyOf(context.accessPathBuilder.accessPath);
slotToAccessPaths.put(slotId,
@@ -146,6 +155,30 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
slotToAccessPaths.put(slotId,
new CollectAccessPathResult(path, context.bottomFilter, ColumnAccessPathType.DATA));
}
+ return null;
+ }
+ // For any other nullable column type (e.g. INT, BIGINT) accessed via IS NULL / IS NOT NULL:
+ // record the [col_name, NULL] path so NestedColumnPruning can emit null-only access paths.
+ // Skip NestedColumnPrunable types (already handled above) and string types (handled above).
+ if (!(dataType instanceof NestedColumnPrunable) && !dataType.isStringLikeType()
+ && !context.accessPathBuilder.isEmpty() && slotReference.nullable()) {
+ context.accessPathBuilder.addPrefix(slotReference.getName());
+ ImmutableList<String> path = ImmutableList.copyOf(context.accessPathBuilder.accessPath);
+ int slotId = slotReference.getExprId().asInt();
+ slotToAccessPaths.put(slotId,
+ new CollectAccessPathResult(path, context.bottomFilter, ColumnAccessPathType.DATA));
+ }
+ // For any other nullable column type accessed directly (not via IS NULL / length / etc.):
+ // record a [col_name] full-access path so that when the column is also used via IS NULL,
+ // stripNullSuffixPaths correctly suppresses the null-only optimization.
+ if (!(dataType instanceof NestedColumnPrunable) && !dataType.isStringLikeType()
+ && !(dataType instanceof VariantType)
+ && context.accessPathBuilder.isEmpty() && slotReference.nullable()) {
+ int slotId = slotReference.getExprId().asInt();
+ slotToAccessPaths.put(slotId,
+ new CollectAccessPathResult(
+ ImmutableList.of(slotReference.getName()),
+ context.bottomFilter, ColumnAccessPathType.DATA));
}
return null;
}
@@ -315,7 +348,19 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
@Override
public Void visitMapKeys(MapKeys mapKeys, CollectorContext context) {
- context = new CollectorContext(context.statementContext, context.bottomFilter);
+ LinkedList<String> suffixPath = context.accessPathBuilder.accessPath;
+ if (isFunctionNullCheckPath(suffixPath)) {
+ // map_keys(nullable_map) returns a NULL array only when the parent map is NULL.
+ // The NULL suffix therefore belongs to the map itself, not to the KEYS child.
+ return continueCollectAccessPath(mapKeys.getArgument(0), context);
+ }
+ if (!suffixPath.isEmpty() && suffixPath.get(0).equals(AccessPathInfo.ACCESS_ALL)) {
+ CollectorContext removeStarContext
+ = new CollectorContext(context.statementContext, context.bottomFilter);
+ removeStarContext.accessPathBuilder.accessPath.addAll(suffixPath.subList(1, suffixPath.size()));
+ removeStarContext.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_KEYS);
+ return continueCollectAccessPath(mapKeys.getArgument(0), removeStarContext);
+ }
context.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_KEYS);
return continueCollectAccessPath(mapKeys.getArgument(0), context);
}
@@ -323,6 +368,11 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
@Override
public Void visitMapValues(MapValues mapValues, CollectorContext context) {
LinkedList<String> suffixPath = context.accessPathBuilder.accessPath;
+ if (isFunctionNullCheckPath(suffixPath)) {
+ // map_values(nullable_map) returns a NULL array only when the parent map is NULL.
+ // A map entry whose value is NULL still produces a non-NULL values array.
+ return continueCollectAccessPath(mapValues.getArgument(0), context);
+ }
if (!suffixPath.isEmpty() && suffixPath.get(0).equals(AccessPathInfo.ACCESS_ALL)) {
CollectorContext removeStarContext
= new CollectorContext(context.statementContext, context.bottomFilter);
@@ -334,6 +384,10 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
return continueCollectAccessPath(mapValues.getArgument(0), context);
}
+ private static boolean isFunctionNullCheckPath(List<String> suffixPath) {
+ return suffixPath.size() == 1 && AccessPathInfo.ACCESS_NULL.equals(suffixPath.get(0));
+ }
+
@Override
public Void visitMapContainsKey(MapContainsKey mapContainsKey, CollectorContext context) {
context.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_KEYS);
@@ -506,14 +560,35 @@ public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void
return visit(arraySortBy, context);
}
- // @Override
- // public Void visitIsNull(IsNull isNull, CollectorContext context) {
- // if (context.accessPathBuilder.isEmpty()) {
- // context.setType(ColumnAccessPathType.META);
- // return continueCollectAccessPath(isNull.child(), context);
- // }
- // return visit(isNull, context);
- // }
+ @Override
+ public Void visitIsNull(IsNull isNull, CollectorContext context) {
+ Expression arg = isNull.child();
+ // Skip variant sub-column paths (v['k'] IS NULL): the sub-column path is already baked
+ // into the SlotReference, so null-only access doesn't apply the same way.
+ if (arg instanceof SlotReference && ((SlotReference) arg).hasSubColPath()) {
+ return visit(isNull, context);
+ }
+ // Optimize IS NULL on nullable expressions: create a context with NULL suffix to indicate
+ // only the null flag is needed. Works for top-level columns (col IS NULL → [col, NULL])
+ // and nested access (struct_element(s, 'city') IS NULL → [s, city, NULL]).
+ // For unrecognized expressions, the default visitor resets context, safely discarding NULL.
+ if (arg.nullable() && context.accessPathBuilder.isEmpty()) {
+ CollectorContext nullContext =
+ new CollectorContext(context.statementContext, context.bottomFilter);
+ nullContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_NULL);
+ return continueCollectAccessPath(arg, nullContext);
+ }
+ return visit(isNull, context);
+ }
+
+ @Override
+ public Void visitNot(Not not, CollectorContext context) {
+ // NOT(IS NULL) == IS NOT NULL: same null-only access pattern
+ if (not.child() instanceof IsNull) {
+ return not.child().accept(this, context);
+ }
+ return visit(not, context);
+ }
private Void collectArrayPathInLambda(Lambda lambda, CollectorContext context) {
List<Expression> arguments = lambda.getArguments();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java
index fffc71ace23..0b361c22760 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java
@@ -30,6 +30,9 @@ public class AccessPathInfo {
// Suffix appended to a string-column path to indicate that only the offset array
// (not the char data) is needed — agreed with BE as the special path component name.
public static final String ACCESS_STRING_OFFSET = "OFFSET";
+ // Suffix appended to a column path to indicate that only the null flag
+ // (not the actual data) is needed — used when the column is only accessed via IS NULL / IS NOT NULL.
+ public static final String ACCESS_NULL = "NULL";
private DataType prunedType;
// allAccessPaths is used to record all access path include predicate access path and non-predicate access path,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
index c738790dc7d..d7ff817f12c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.rewrite;
+import org.apache.doris.analysis.ColumnAccessPathType;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectorContext;
@@ -72,7 +73,8 @@ public class AccessPathPlanCollector extends DefaultPlanVisitor<Void, StatementC
private boolean shouldCollectAccessPath(Slot slot) {
return slot.getDataType() instanceof NestedColumnPrunable
|| slot.getDataType().isVariantType()
- || slot.getDataType().isStringLikeType();
+ || slot.getDataType().isStringLikeType()
+ || slot.nullable();
}
@Override
@@ -347,7 +349,7 @@ public class AccessPathPlanCollector extends DefaultPlanVisitor<Void, StatementC
}
Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
if (!accessPaths.isEmpty()) {
- scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ scanSlotToAccessPaths.put(slot, normalizeDataSkippingOnlyAccessPaths(accessPaths));
}
}
return null;
@@ -361,7 +363,7 @@ public class AccessPathPlanCollector extends DefaultPlanVisitor<Void, StatementC
}
Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
if (!accessPaths.isEmpty()) {
- scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ scanSlotToAccessPaths.put(slot, normalizeDataSkippingOnlyAccessPaths(accessPaths));
}
}
return null;
@@ -388,4 +390,33 @@ public class AccessPathPlanCollector extends DefaultPlanVisitor<Void, StatementC
exprCollector.collect(expression);
}
}
+
+ static List<CollectAccessPathResult> normalizeDataSkippingOnlyAccessPaths(
+ Collection<CollectAccessPathResult> accessPaths) {
+ List<CollectAccessPathResult> normalizedAccessPaths = new ArrayList<>();
+ for (CollectAccessPathResult accessPath : accessPaths) {
+ List<String> path = accessPath.getPath();
+ if (isDataSkippingOnlyAccessPath(path) && path.size() > 1) {
+ // NULL/OFFSET suffixes are OLAP segment-reader-only optimizations. External
+ // table and TVF readers use access paths as real nested field paths, so read
+ // the referenced column/sub-column normally instead of sending a pseudo field.
+ normalizedAccessPaths.add(new CollectAccessPathResult(
+ new ArrayList<>(path.subList(0, path.size() - 1)),
+ accessPath.isPredicate(),
+ ColumnAccessPathType.DATA));
+ } else {
+ normalizedAccessPaths.add(accessPath);
+ }
+ }
+ return normalizedAccessPaths;
+ }
+
+ private static boolean isDataSkippingOnlyAccessPath(List<String> path) {
+ if (path.isEmpty()) {
+ return false;
+ }
+ String lastComponent = path.get(path.size() - 1);
+ return AccessPathInfo.ACCESS_NULL.equals(lastComponent)
+ || AccessPathInfo.ACCESS_STRING_OFFSET.equals(lastComponent);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
index ab33926fa1e..0da73d3e936 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -25,6 +25,8 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Cardinality;
@@ -34,6 +36,7 @@ import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.MapType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
import org.apache.doris.nereids.types.NullType;
import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
@@ -83,13 +86,14 @@ public class NestedColumnPruning implements CustomRewriter {
if (!sessionVariable.enablePruneNestedColumns
|| (!statementContext.hasNestedColumns()
&& !containsVariant(plan)
- && !(containsStringLength(plan)))) {
+ && !containsStringLength(plan)
+ && !containsNullCheck(plan))) {
return plan;
}
-
AccessPathPlanCollector collector = new AccessPathPlanCollector();
Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths = collector.collect(plan, statementContext);
- Map<Integer, AccessPathInfo> slotToResult = pruneDataType(slotToAccessPaths);
+ Map<Integer, AccessPathInfo> slotToResult = pruneDataType(slotToAccessPaths,
+ jobContext.getCascadesContext().isMaterializedViewRewritePlanFragment());
if (!slotToResult.isEmpty()) {
Map<Integer, AccessPathInfo> slotIdToPruneType = Maps.newLinkedHashMap();
@@ -164,8 +168,43 @@ public class NestedColumnPruning implements CustomRewriter {
return hasVariant.get();
}
+ /** Returns true when the plan tree contains IS NULL or IS NOT NULL on a nullable slot. */
+ private static boolean containsNullCheck(Plan plan) {
+ AtomicBoolean found = new AtomicBoolean(false);
+ plan.foreachUp(node -> {
+ if (found.get()) {
+ return;
+ }
+ Plan current = (Plan) node;
+ for (Expression expression : current.getExpressions()) {
+ if (expressionContainsNullCheck(expression)) {
+ found.set(true);
+ return;
+ }
+ }
+ });
+ return found.get();
+ }
+
+ private static boolean expressionContainsNullCheck(Expression expr) {
+ if (expr instanceof IsNull && expr.child(0).nullable()) {
+ return true;
+ }
+ if (expr instanceof Not && expr.child(0) instanceof IsNull
+ && expr.child(0).child(0).nullable()) {
+ return true;
+ }
+ for (Expression child : expr.children()) {
+ if (expressionContainsNullCheck(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private static Map<Integer, AccessPathInfo> pruneDataType(
- Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths) {
+ Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths,
+ boolean skipDataSkippingOnlyAccessPath) {
Map<Integer, AccessPathInfo> result = new LinkedHashMap<>();
Map<Slot, DataTypeAccessTree> slotIdToAllAccessTree = new LinkedHashMap<>();
Map<Slot, DataTypeAccessTree> slotIdToPredicateAccessTree = new LinkedHashMap<>();
@@ -197,6 +236,19 @@ public class NestedColumnPruning implements CustomRewriter {
}
continue;
}
+ if (skipDataSkippingOnlyAccessPath
+ && containsDataSkippingOnlyAccessPath(collectAccessPathResults)) {
+ // An MV rewrite child context optimizes a temporary plan fragment rather
+ // than the final plan. A nested metadata-only path such as
+ // [s, city, NULL] or [s, city, OFFSET] would otherwise prune the scan slot
+ // to only that nested field, while the final MV rewritten plan may still
+ // reuse the same slot as a full complex value or need another child. Drop
+ // access-info for the whole slot instead of just removing that path:
+ // predicate expressions inside this fragment still reference the original
+ // slot shape, so partial pruning after deleting the predicate-only path
+ // could make the fragment itself inconsistent.
+ continue;
+ }
for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) {
List<String> path = collectAccessPathResult.getPath();
ColumnAccessPathType pathType = collectAccessPathResult.getType();
@@ -226,8 +278,21 @@ public class NestedColumnPruning implements CustomRewriter {
if (slot.getDataType().isStringLikeType()) {
if (accessTree.hasStringOffsetOnlyAccess()) {
+ if (skipDataSkippingOnlyAccessPath) {
+ continue;
+ }
// Offset-only access (e.g. length(str_col)): type stays varchar,
// but we must still send the access path to BE so it skips the char data.
+ stripNullSuffixPaths(slot, allAccessPaths);
+ List<ColumnAccessPath> allPaths = buildColumnAccessPaths(slot, allAccessPaths);
+ result.put(slot.getExprId().asInt(),
+ new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>()));
+ } else if (accessTree.hasNullCheckOnlyAccess()) {
+ if (skipDataSkippingOnlyAccessPath) {
+ continue;
+ }
+ // Null-check-only access (e.g. str_col IS NULL): type stays varchar,
+ // but we send [col, NULL] access path so BE only reads the null flag.
List<ColumnAccessPath> allPaths = buildColumnAccessPaths(slot, allAccessPaths);
result.put(slot.getExprId().asInt(),
new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>()));
@@ -238,6 +303,9 @@ public class NestedColumnPruning implements CustomRewriter {
if ((slot.getDataType().isArrayType() || slot.getDataType().isMapType())
&& accessTree.hasStringOffsetOnlyAccess()) {
+ if (skipDataSkippingOnlyAccessPath) {
+ continue;
+ }
// Offset-only access (e.g. length(arr_col) / length(map_col)): type stays unchanged,
// but we must send the OFFSET access path to BE so it skips element/key-value data.
List<ColumnAccessPath> allPaths = buildColumnAccessPaths(slot, allAccessPaths);
@@ -246,7 +314,22 @@ public class NestedColumnPruning implements CustomRewriter {
continue;
}
+ // Null-check-only access (e.g. col IS NULL / col IS NOT NULL): type stays unchanged,
+ // but we must send the [col, NULL] access path to BE so it only reads the null flag.
+ if (accessTree.hasNullCheckOnlyAccess()) {
+ if (skipDataSkippingOnlyAccessPath) {
+ continue;
+ }
+ List<ColumnAccessPath> allPaths = buildColumnAccessPaths(slot, allAccessPaths);
+ result.put(slot.getExprId().asInt(),
+ new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>()));
+ continue;
+ }
+
if (slot.getDataType().isMapType() && accessTree.hasMapValueOffsetOnlyAccess()) {
+ if (skipDataSkippingOnlyAccessPath) {
+ continue;
+ }
// length(map_col['key']): keys read in full (element lookup) + values offset-only.
// Emit [col, KEYS] and [col, VALUES, OFFSET] directly instead of the collected
// [col, *, OFFSET] path which the BE cannot interpret for split key/value access.
@@ -294,7 +377,14 @@ public class NestedColumnPruning implements CustomRewriter {
}
paths.removeAll(pathsToRemove);
paths.addAll(pathsToAdd);
+
+ // Strip NULL-suffix paths when a non-NULL path also exists for the same slot.
+ // E.g. `SELECT col FROM t WHERE col IS NULL` — full data is needed, NULL path is redundant.
+ stripNullSuffixPaths(slot, allAccessPaths);
List<ColumnAccessPath> allPaths = buildColumnAccessPaths(slot, allAccessPaths);
+ if (shouldSkipAccessInfo(slot, prunedDataType, allPaths, predicateAccessPaths)) {
+ continue;
+ }
result.put(slot.getExprId().asInt(),
new AccessPathInfo(prunedDataType, allPaths, new ArrayList<>()));
}
@@ -309,7 +399,7 @@ public class NestedColumnPruning implements CustomRewriter {
// third: build predicate access path
for (Entry<Slot, DataTypeAccessTree> kv : slotIdToPredicateAccessTree.entrySet()) {
Slot slot = kv.getKey();
-
+ stripNullSuffixPaths(slot, predicateAccessPaths);
List<ColumnAccessPath> predicatePaths =
buildColumnAccessPaths(slot, predicateAccessPaths);
AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt());
@@ -331,6 +421,25 @@ public class NestedColumnPruning implements CustomRewriter {
return result;
}
+ private static boolean containsDataSkippingOnlyAccessPath(
+ List<CollectAccessPathResult> collectAccessPathResults) {
+ for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) {
+ if (isDataSkippingOnlyAccessPath(collectAccessPathResult.getPath())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isDataSkippingOnlyAccessPath(List<String> path) {
+ if (path.isEmpty()) {
+ return false;
+ }
+ String lastComponent = path.get(path.size() - 1);
+ return AccessPathInfo.ACCESS_NULL.equals(lastComponent)
+ || AccessPathInfo.ACCESS_STRING_OFFSET.equals(lastComponent);
+ }
+
/**
* Decide whether an OFFSET-suffix path can be removed because another non-OFFSET path
* already covers the same container.
@@ -480,6 +589,76 @@ public class NestedColumnPruning implements CustomRewriter {
}
}
+ /**
+ * Strip NULL-suffix paths that are redundant because a non-NULL path reads the same
+ * column/subcolumn in full (its data inherently includes the null flag).
+ *
+ * For example, [int_col, NULL] is removed when [int_col] exists — reading the full
+ * column includes its null flag.
+ *
+ * A parent NULL path must also be removed when any child path is required under the
+ * same prefix, e.g. [struct_col, NULL] with [struct_col, city]. This looks like the
+ * parent null map is still useful for predicates, but it cannot be kept in
+ * allAccessPaths with the current BE iterator contract: Struct/Array/Map iterators
+ * treat a leading NULL sub-path as NULL_MAP_ONLY and skip all children. If FE kept
+ * [struct_col.NULL, struct_col.city] in allAccessPaths, BE would read only the
+ * struct null map and default-fill city instead of routing the city child iterator.
+ * predicateAccessPaths still retains the NULL path, while the normal nullable
+ * container read materializes the parent null map together with required children.
+ */
+ private static void stripNullSuffixPaths(
+ Slot slot, Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> allAccessPaths) {
+ int slotId = slot.getExprId().asInt();
+ Collection<Pair<ColumnAccessPathType, List<String>>> slotPaths = allAccessPaths.get(slotId);
+
+ List<Pair<ColumnAccessPathType, List<String>>> toRemove = new ArrayList<>();
+ for (Pair<ColumnAccessPathType, List<String>> p : slotPaths) {
+ List<String> path = p.second;
+ if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) {
+ continue;
+ }
+ // Prefix is the column/subcolumn path without the trailing NULL suffix.
+ // A non-NULL path that equals this prefix means the same column/subcolumn
+ // is read in full, making the NULL-only path redundant.
+ // An OFFSET-suffix path over the same prefix is also enough for the BE to
+ // derive null-ness for variable-length columns, so [col.NULL] is redundant
+ // when [col.OFFSET] already exists.
+ List<String> prefix = path.subList(0, path.size() - 1);
+ boolean covered = false;
+ for (Pair<ColumnAccessPathType, List<String>> q : slotPaths) {
+ List<String> other = q.second;
+ if (other.isEmpty()
+ || AccessPathInfo.ACCESS_NULL.equals(other.get(other.size() - 1))) {
+ continue;
+ }
+ if (other.equals(prefix)) {
+ covered = true;
+ break;
+ }
+ if (hasStrictPrefix(other, prefix)) {
+ covered = true;
+ break;
+ }
+ if (other.size() == prefix.size() + 1
+ && AccessPathInfo.ACCESS_STRING_OFFSET.equals(other.get(other.size() - 1))
+ && other.subList(0, prefix.size()).equals(prefix)) {
+ covered = true;
+ break;
+ }
+ }
+ if (covered) {
+ toRemove.add(p);
+ }
+ }
+ for (Pair<ColumnAccessPathType, List<String>> r : toRemove) {
+ allAccessPaths.remove(slotId, r);
+ }
+ }
+
+ private static boolean hasStrictPrefix(List<String> path, List<String> prefix) {
+ return path.size() > prefix.size() && path.subList(0, prefix.size()).equals(prefix);
+ }
+
private static List<ColumnAccessPath> buildColumnAccessPaths(
Slot slot, Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> accessPaths) {
List<ColumnAccessPath> paths = new ArrayList<>();
@@ -508,6 +687,25 @@ public class NestedColumnPruning implements CustomRewriter {
return paths;
}
+ private static boolean shouldSkipAccessInfo(
+ Slot slot, DataType prunedDataType, List<ColumnAccessPath> allPaths,
+ Multimap<Integer, Pair<ColumnAccessPathType, List<String>>> predicateAccessPaths) {
+ if (!prunedDataType.equals(slot.getDataType())) {
+ return false;
+ }
+ if (slot.getDataType() instanceof NestedColumnPrunable || slot.getDataType().isVariantType()) {
+ return false;
+ }
+ if (!predicateAccessPaths.get(slot.getExprId().asInt()).isEmpty()) {
+ return false;
+ }
+ if (allPaths.size() != 1) {
+ return false;
+ }
+ List<String> path = allPaths.get(0).getPath();
+ return path.size() == 1;
+ }
+
/** DataTypeAccessTree */
public static class DataTypeAccessTree {
// type of this level
@@ -522,6 +720,10 @@ public class NestedColumnPruning implements CustomRewriter {
// When this flag is set and accessAll is NOT set, pruneDataType() returns BigIntType
// to signal that the BE only needs to read the offset array, not the chars data.
private boolean isStringOffsetOnly;
+ // True when this column node is accessed ONLY via IS NULL / IS NOT NULL.
+ // When this flag is set and accessAll is NOT set, the BE only needs to read the null flag,
+ // not the actual column data.
+ private boolean isNullCheckOnly;
// for the future, only access the meta of the column,
// e.g. `is not null` can only access the column's offset, not need to read the data
private ColumnAccessPathType pathType;
@@ -629,6 +831,17 @@ public class NestedColumnPruning implements CustomRewriter {
return type.isStringLikeType() && isStringOffsetOnly && !accessAll;
}
+ /** True when the column is accessed ONLY via IS NULL / IS NOT NULL,
+ * meaning the BE only needs to read the null flag, not the actual data. */
+ public boolean hasNullCheckOnlyAccess() {
+ if (isRoot) {
+ DataTypeAccessTree child = children.values().iterator().next();
+ return child.isNullCheckOnly && !child.accessAll
+ && !child.isStringOffsetOnly && !child.accessPartialChild;
+ }
+ return isNullCheckOnly && !accessAll && !isStringOffsetOnly && !accessPartialChild;
+ }
+
/** pruneCastType */
public DataType pruneCastType(DataTypeAccessTree origin, DataTypeAccessTree cast) {
if (type instanceof StructType) {
@@ -715,14 +928,22 @@ public class NestedColumnPruning implements CustomRewriter {
if (accessIndex >= path.size()) {
accessAll = true;
return;
- } else {
- accessPartialChild = true;
}
if (pathType == ColumnAccessPathType.DATA) {
this.pathType = ColumnAccessPathType.DATA;
}
+ // NULL path component: the column is accessed only via IS NULL / IS NOT NULL.
+ // Mark null-check-only and return without setting accessAll or accessPartialChild,
+ // so that parent nodes can distinguish "null-only leaf" from "has real sub-access".
+ if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_NULL)) {
+ isNullCheckOnly = true;
+ return;
+ }
+
+ accessPartialChild = true;
+
if (this.type.isStructType()) {
String fieldName = path.get(accessIndex).toLowerCase();
DataTypeAccessTree child = children.get(fieldName);
@@ -761,10 +982,12 @@ public class NestedColumnPruning implements CustomRewriter {
valuesChild.setAccessByPath(path, accessIndex + 1, pathType);
return;
} else if (fieldName.equals(AccessPathInfo.ACCESS_MAP_KEYS)) {
- // only access the keys and not need enter keys, because it must be primitive type.
- // e.g. map_keys(map_column)
+ // Access the keys sub-column. Delegate to child so that trailing path
+ // components (e.g. NULL for IS NULL) are processed correctly.
+ // When no trailing component exists, setAccessByPath reaches end-of-path
+ // and sets accessAll = true, preserving the original behavior.
DataTypeAccessTree keysChild = children.get(AccessPathInfo.ACCESS_MAP_KEYS);
- keysChild.accessAll = true;
+ keysChild.setAccessByPath(path, accessIndex + 1, pathType);
return;
} else if (fieldName.equals(AccessPathInfo.ACCESS_MAP_VALUES)) {
// only access the values without keys, and maybe prune the value's data type.
@@ -824,6 +1047,11 @@ public class NestedColumnPruning implements CustomRewriter {
} else if (isStringOffsetOnly && !accessPartialChild) {
// Only the offset array is accessed (e.g. length(str_col)).
return Optional.of(type);
+ } else if (isNullCheckOnly && !accessPartialChild) {
+ // Only the null flag is accessed (e.g. col IS NULL / struct_element(s,'f') IS NULL).
+ // Return the node's type so that parent nodes include this child in their pruned type,
+ // while the access path (ending in NULL) tells BE to skip actual data reading.
+ return Optional.of(type);
} else if (!accessPartialChild) {
return Optional.empty();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
index cffb6fd7776..6a8fd28b902 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -679,7 +679,8 @@ public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
int slotId = slot.getExprId().asInt();
if ((slot.getDataType() instanceof NestedColumnPrunable
|| slot.getDataType().isVariantType()
- || slot.getDataType().isStringLikeType())
+ || slot.getDataType().isStringLikeType()
+ || slot.nullable())
&& replacedDataTypes.containsKey(slotId)) {
shouldReplaceSlots.add(slotId);
shouldPrune = true;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 070876d79b7..8d8d3441fa6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -25,6 +25,7 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.Triple;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
@@ -40,6 +41,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
+import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.NestedColumnPrunable;
import org.apache.doris.nereids.types.NullType;
@@ -60,6 +62,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Optional;
import java.util.TreeSet;
import java.util.function.Consumer;
@@ -472,61 +475,91 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
public void testFilter() throws Throwable {
assertColumn("select 100 from tbl where s is not null",
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
- ImmutableList.of(path("s")),
- ImmutableList.of(path("s"))
+ ImmutableList.of(path("s", "NULL")),
+ ImmutableList.of(path("s", "NULL"))
);
+ // The IF expression itself is not collected as a null-only parent access here; the
+ // struct_element predicate still lets NCP prune the scan slot to the city field.
assertColumn("select 100 from tbl where if(id = 1, null, s) is not null or struct_element(s, 'city') = 'beijing'",
- "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
- ImmutableList.of(path("s")),
- ImmutableList.of(path("s"))
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of(path("s", "city"))
);
assertColumn("select 100 from tbl where struct_element(s, 'city') is not null",
"struct<city:text>",
- ImmutableList.of(path("s", "city")),
- ImmutableList.of(path("s", "city"))
+ ImmutableList.of(path("s", "city", "NULL")),
+ ImmutableList.of(path("s", "city", "NULL"))
);
assertColumn("select 100 from tbl where struct_element(s, 'data') is not null",
"struct<data:array<map<int,struct<a:int,b:double>>>>",
- ImmutableList.of(path("s", "data")),
- ImmutableList.of(path("s", "data"))
+ ImmutableList.of(path("s", "data", "NULL")),
+ ImmutableList.of(path("s", "data", "NULL"))
);
assertColumn("select 100 from tbl where struct_element(s, 'data')[1] is not null",
"struct<data:array<map<int,struct<a:int,b:double>>>>",
- ImmutableList.of(path("s", "data", "*")),
- ImmutableList.of(path("s", "data", "*"))
+ ImmutableList.of(path("s", "data", "*", "NULL")),
+ ImmutableList.of(path("s", "data", "*", "NULL"))
);
assertColumn("select 100 from tbl where map_keys(struct_element(s, 'data')[1]) is not null",
"struct<data:array<map<int,struct<a:int,b:double>>>>",
- ImmutableList.of(path("s", "data", "*", "KEYS")),
- ImmutableList.of(path("s", "data", "*", "KEYS"))
+ ImmutableList.of(path("s", "data", "*", "NULL")),
+ ImmutableList.of(path("s", "data", "*", "NULL"))
);
assertColumn("select 100 from tbl where map_values(struct_element(s, 'data')[1]) is not null",
"struct<data:array<map<int,struct<a:int,b:double>>>>",
- ImmutableList.of(path("s", "data", "*", "VALUES")),
- ImmutableList.of(path("s", "data", "*", "VALUES"))
+ ImmutableList.of(path("s", "data", "*", "NULL")),
+ ImmutableList.of(path("s", "data", "*", "NULL"))
);
assertColumn("select 100 from tbl where struct_element(map_values(struct_element(s, 'data')[1])[1], 'a') is not null",
"struct<data:array<map<int,struct<a:int>>>>",
- ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
- ImmutableList.of(path("s", "data", "*", "VALUES", "a"))
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a", "NULL")),
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a", "NULL"))
);
assertColumn("select 100 from tbl where struct_element(s, 'data')[1][1] is not null",
"struct<data:array<map<int,struct<a:int,b:double>>>>",
- ImmutableList.of(path("s", "data", "*", "*")),
- ImmutableList.of(path("s", "data", "*", "*"))
+ ImmutableList.of(path("s", "data", "*", "*", "NULL")),
+ ImmutableList.of(path("s", "data", "*", "*", "NULL"))
);
assertColumn("select 100 from tbl where struct_element(struct_element(s, 'data')[1][1], 'a') is not null",
"struct<data:array<map<int,struct<a:int>>>>",
- ImmutableList.of(path("s", "data", "*", "*", "a")),
- ImmutableList.of(path("s", "data", "*", "*", "a"))
+ ImmutableList.of(path("s", "data", "*", "*", "a", "NULL")),
+ ImmutableList.of(path("s", "data", "*", "*", "a", "NULL"))
);
assertColumn("select 100 from tbl where struct_element(struct_element(s, 'data')[1][1], 'b') is not null",
"struct<data:array<map<int,struct<b:double>>>>",
- ImmutableList.of(path("s", "data", "*", "*", "b")),
- ImmutableList.of(path("s", "data", "*", "*", "b"))
+ ImmutableList.of(path("s", "data", "*", "*", "b", "NULL")),
+ ImmutableList.of(path("s", "data", "*", "*", "b", "NULL"))
+ );
+ }
+
+ @Test
+ public void testMapKeysAndValuesFunctionNullCheckUseParentMapNullPath() throws Exception {
+ // map_keys/map_values are PropagateNullable functions: the returned array is NULL only
+ // when the input map is NULL. Their function-level IS NULL predicates must therefore
+ // read the parent map null map, not the KEYS/VALUES child null maps.
+ assertColumn("select 100 from str_tbl where map_keys(map_col) is null",
+ "map<text,text>",
+ ImmutableList.of(path("map_col", "NULL")),
+ ImmutableList.of(path("map_col", "NULL"))
+ );
+ assertColumn("select 100 from str_tbl where map_values(map_col) is null",
+ "map<text,text>",
+ ImmutableList.of(path("map_col", "NULL")),
+ ImmutableList.of(path("map_col", "NULL"))
+ );
+
+ assertColumn("select map_keys(map_col) from str_tbl where map_keys(map_col) is null",
+ "map<text,text>",
+ ImmutableList.of(path("map_col", "KEYS")),
+ ImmutableList.of(path("map_col", "NULL"))
+ );
+ assertColumn("select map_values(map_col) from str_tbl where map_values(map_col) is null",
+ "map<text,text>",
+ ImmutableList.of(path("map_col", "VALUES")),
+ ImmutableList.of(path("map_col", "NULL"))
);
}
@@ -535,19 +568,19 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
assertColumn("select s from tbl where struct_element(s, 'city') is not null",
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
ImmutableList.of(path("s")),
- ImmutableList.of(path("s", "city"))
+ ImmutableList.of(path("s", "city", "NULL"))
);
assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null",
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
- ImmutableList.of(path("s", "data"), path("s", "city")),
- ImmutableList.of(path("s", "city"))
+ ImmutableList.of(path("s", "city", "NULL"), path("s", "data")),
+ ImmutableList.of(path("s", "city", "NULL"))
);
assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null and struct_element(s, 'data') is not null",
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
- ImmutableList.of(path("s", "data"), path("s", "city")),
- ImmutableList.of(path("s", "data"), path("s", "city"))
+ ImmutableList.of(path("s", "city", "NULL"), path("s", "data")),
+ ImmutableList.of(path("s", "city", "NULL"), path("s", "data", "NULL"))
);
}
@@ -1221,92 +1254,243 @@ public class PruneNestedColumnTest extends TestWithFeService implements MemoPatt
// @Test
// public void testStringLengthPruning() {
- // // ── Case 1: length(str_col) only ─ offset-only optimization applied ──────────
- // assertStringColumn(
- // "select length(str_col) from str_tbl",
- // "str_col",
- // true,
- // ImmutableList.of(path("str_col", "offset")));
-
- // // ── Case 2: length(str_col) + direct projection of str_col ─ suppressed ─────
- // assertStringColumn(
- // "select length(str_col), str_col from str_tbl",
- // "str_col",
- // false,
- // ImmutableList.of());
-
- // // ── Case 3: length(str_col) + substr(str_col, …) ─ suppressed ───────────────
- // assertStringColumn(
- // "select length(str_col), substr(str_col, 2) from str_tbl",
- // "str_col",
- // false,
- // ImmutableList.of());
-
- // // ── Case 4: length applied to a struct field ─ struct pruned to bigint field ─
- // // c_struct has {f1:int, f3:string}; only f3 accessed offset-only →
- // // pruned type is struct<f3:bigint>, access path is DATA(["c_struct","f3","offset"])
- // assertColumn(
- // "select length(struct_element(c_struct, 'f3')) from str_tbl",
- // "struct<f3:bigint>",
- // ImmutableList.of(path("c_struct", "f3", "offset")),
- // ImmutableList.of());
-
- // // ── Case 5: length(struct field) + direct read of same field ─ suppressed ───
- // // Both the full-data path ["c_struct","f3"] and offset path ["c_struct","f3","offset"]
- // // are recorded; f3 pruneDataType() sees accessAll=true → returns text (not bigint).
- // assertColumn(
- // "select length(struct_element(c_struct, 'f3')), struct_element(c_struct, 'f3') from str_tbl",
- // "struct<f3:text>",
- // ImmutableList.of(path("c_struct", "f3"), path("c_struct", "f3", "offset")),
- // ImmutableList.of());
- // }
-
- // /**
- // * Verify that a specific string-typed column in the rewritten LogicalOlapScan either has
- // * BigIntType (offset-only optimization applied) or retains its original string type (suppressed).
- // *
- // * @param sql query to analyze and rewrite
- // * @param columnName name of the string column to inspect
- // * @param expectOptimized true → expect BigIntType + access paths; false → expect string type
- // * @param expectAllPaths expected access paths when {@code expectOptimized} is true
- // */
- // private void assertStringColumn(String sql, String columnName,
- // boolean expectOptimized, List<ColumnAccessPath> expectAllPaths) {
- // Plan rewritePlan = PlanChecker.from(connectContext)
- // .analyze(sql)
- // .rewrite()
- // .getCascadesContext()
- // .getRewritePlan();
-
- // LogicalOlapScan scan = rewritePlan.collect(LogicalOlapScan.class::isInstance)
- // .stream()
- // .map(p -> (LogicalOlapScan) p)
- // .findFirst()
- // .orElseThrow(() -> new AssertionError("No LogicalOlapScan in plan for: " + sql));
-
- // for (Slot slot : scan.getOutput()) {
- // if (!slot.getName().equalsIgnoreCase(columnName)) {
- // continue;
- // }
- // SlotReference slotRef = (SlotReference) slot;
- // if (expectOptimized) {
- // Assertions.assertEquals(BigIntType.INSTANCE, slotRef.getDataType(),
- // "Slot '" + columnName + "' should be BigIntType after offset-only optimization");
- // Optional<List<ColumnAccessPath>> allPaths = slotRef.getAllAccessPaths();
- // Assertions.assertTrue(allPaths.isPresent() && !allPaths.get().isEmpty(),
- // "Slot '" + columnName + "' should have access paths set");
- // Assertions.assertEquals(
- // new TreeSet<>(expectAllPaths),
- // new TreeSet<>(allPaths.get()),
- // "Unexpected access paths for slot '" + columnName + "'");
- // } else {
- // Assertions.assertNotEquals(BigIntType.INSTANCE, slotRef.getDataType(),
- // "Slot '" + columnName + "' should NOT be BigIntType (optimization suppressed)");
- // }
- // return;
- // }
- // Assertions.fail("Column '" + columnName + "' not found in LogicalOlapScan output for: " + sql);
- // }
+
+ @Test
+ public void testStructIsNullPruning() throws Exception {
+ // struct column IS NULL → null-only access, emit [s, NULL] path, type stays struct
+ assertColumn("select 1 from tbl where s is null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "NULL")),
+ ImmutableList.of(path("s", "NULL")));
+ }
+
+ @Test
+ public void testStructIsNotNullPruning() throws Exception {
+ // struct column IS NOT NULL → same null-only access pattern
+ assertColumn("select 1 from tbl where s is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "NULL")),
+ ImmutableList.of(path("s", "NULL")));
+ }
+
+ @Test
+ public void testStructIsNullMixedAccess() throws Exception {
+ // Parent NULL path must be stripped from allPaths when a child path is also required.
+ // Otherwise BE StructFileColumnIterator sees the parent NULL sub-path first, switches
+ // the whole struct iterator to NULL_MAP_ONLY, and skips the child iterator.
+ // predicateAccessPaths keeps [s, NULL] because the predicate itself still uses it.
+ assertColumn("select struct_element(s, 'city') from tbl where s is null",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of(path("s", "NULL")));
+
+ // This shape is closer to the production bug: one predicate needs the parent
+ // null map, another predicate needs a child null map, and the projection needs
+ // a different child data path. The parent [s.NULL] cannot remain in allPaths
+ // with [s.data], but both predicate NULL paths must remain in predicate paths.
+ assertColumn("select struct_element(s, 'data') from tbl "
+ + "where s is null or struct_element(s, 'city') is null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "city", "NULL"), path("s", "data")),
+ ImmutableList.of(path("s", "NULL"), path("s", "city", "NULL")));
+ }
+
+ @Test
+ public void testStringLengthPruning() throws Exception {
+ // ── Case 1: length(str_col) only ─ offset-only optimization applied ──────────
+ assertStringColumn(
+ "select length(str_col) from str_tbl",
+ "str_col",
+ true,
+ ImmutableList.of(path("str_col", "OFFSET")));
+
+ // ── Case 2: length(str_col) + direct projection of str_col ─ suppressed ─────
+ assertStringColumn(
+ "select length(str_col), str_col from str_tbl",
+ "str_col",
+ false,
+ ImmutableList.of());
+
+ // ── Case 3: length(str_col) + substr(str_col, …) ─ suppressed ───────────────
+ assertStringColumn(
+ "select length(str_col), substr(str_col, 2) from str_tbl",
+ "str_col",
+ false,
+ ImmutableList.of());
+
+ // ── Case 4: length applied to a struct field ─ struct pruned to bigint field ─
+ // c_struct has {f1:int, f3:string}; only f3 accessed offset-only →
+ // pruned type is struct<f3:bigint>, access path is DATA(["c_struct","f3","offset"])
+ assertColumn(
+ "select length(struct_element(c_struct, 'f3')) from str_tbl",
+ "struct<f3:text>",
+ ImmutableList.of(path("c_struct", "f3", "OFFSET")),
+ ImmutableList.of());
+
+ // ── Case 5: length(struct field) + direct read of same field ─ suppressed ───
+ // Both the full-data path ["c_struct","f3"] and offset path ["c_struct","f3","offset"]
+ // are recorded; f3 pruneDataType() sees accessAll=true → returns text (not bigint).
+ assertColumn(
+ "select length(struct_element(c_struct, 'f3')), struct_element(c_struct, 'f3') from str_tbl",
+ "struct<f3:text>",
+ ImmutableList.of(path("c_struct", "f3")),
+ ImmutableList.of());
+ }
+
+ @Test
+ public void testNonOlapDataSkippingOnlyAccessPathFallback() {
+ List<CollectAccessPathResult> normalizedAccessPaths =
+ AccessPathPlanCollector.normalizeDataSkippingOnlyAccessPaths(ImmutableList.of(
+ new CollectAccessPathResult(
+ ImmutableList.of("s", "city", "NULL"), true, ColumnAccessPathType.DATA),
+ new CollectAccessPathResult(
+ ImmutableList.of("array_column", "OFFSET"), false, ColumnAccessPathType.DATA),
+ new CollectAccessPathResult(
+ ImmutableList.of("s", "city"), false, ColumnAccessPathType.DATA)));
+
+ Assertions.assertEquals(3, normalizedAccessPaths.size());
+ Assertions.assertEquals(ImmutableList.of("s", "city"), normalizedAccessPaths.get(0).getPath());
+ Assertions.assertTrue(normalizedAccessPaths.get(0).isPredicate());
+ Assertions.assertEquals(ColumnAccessPathType.DATA, normalizedAccessPaths.get(0).getType());
+ Assertions.assertEquals(ImmutableList.of("array_column"), normalizedAccessPaths.get(1).getPath());
+ Assertions.assertFalse(normalizedAccessPaths.get(1).isPredicate());
+ Assertions.assertEquals(ColumnAccessPathType.DATA, normalizedAccessPaths.get(1).getType());
+ Assertions.assertEquals(ImmutableList.of("s", "city"), normalizedAccessPaths.get(2).getPath());
+ }
+
+ @Test
+ public void testMvRewritePlanFragmentSkipsNullOnlyAccessPath() {
+ SlotReference normalSlot = rewriteAndFindScanSlot(
+ "select 1 from str_tbl where str_col is not null", "str_col", false);
+ Assertions.assertEquals(
+ new TreeSet<>(ImmutableList.of(path("str_col", "NULL"))),
+ new TreeSet<>(normalSlot.getAllAccessPaths().get()));
+ Assertions.assertEquals(
+ new TreeSet<>(ImmutableList.of(path("str_col", "NULL"))),
+ new TreeSet<>(normalSlot.getPredicateAccessPaths().get()));
+
+ SlotReference fragmentSlot = rewriteAndFindScanSlot(
+ "select 1 from str_tbl where str_col is not null", "str_col", true);
+ assertNoAccessPaths(fragmentSlot);
+
+ SlotReference nestedNormalSlot = rewriteAndFindScanSlot(
+ "select 1 from tbl where struct_element(s, 'city') is not null", "s", false);
+ Assertions.assertEquals(
+ new TreeSet<>(ImmutableList.of(path("s", "city", "NULL"))),
+ new TreeSet<>(nestedNormalSlot.getAllAccessPaths().get()));
+ Assertions.assertEquals(
+ new TreeSet<>(ImmutableList.of(path("s", "city", "NULL"))),
+ new TreeSet<>(nestedNormalSlot.getPredicateAccessPaths().get()));
+
+ // MV rewrite optimizes temporary fragments whose later consumers are not visible.
+ // If the fragment only needs nested null metadata, e.g. [s.city.NULL], pruning the
+ // scan slot to struct<city:...> can break the final rewritten MV plan when it still
+ // needs the full struct or another child. The fragment marker therefore suppresses
+ // nested null-only access info too, not just top-level [col.NULL].
+ SlotReference nestedFragmentSlot = rewriteAndFindScanSlot(
+ "select 1 from tbl where struct_element(s, 'city') is not null", "s", true);
+ assertNoAccessPaths(nestedFragmentSlot);
+ }
+
+ @Test
+ public void testMvRewritePlanFragmentSkipsOffsetOnlyAccessPath() {
+ SlotReference normalSlot = rewriteAndFindScanSlot(
+ "select 1 from str_tbl where length(str_col) > 0", "str_col", false);
+ Assertions.assertEquals(
+ new TreeSet<>(ImmutableList.of(path("str_col", "OFFSET"))),
+ new TreeSet<>(normalSlot.getAllAccessPaths().get()));
+ Assertions.assertEquals(
+ new TreeSet<>(ImmutableList.of(path("str_col", "OFFSET"))),
+ new TreeSet<>(normalSlot.getPredicateAccessPaths().get()));
+
+ SlotReference fragmentSlot = rewriteAndFindScanSlot(
+ "select 1 from str_tbl where length(str_col) > 0", "str_col", true);
+ assertNoAccessPaths(fragmentSlot);
+
+ SlotReference nestedNormalSlot = rewriteAndFindScanSlot(
+ "select 1 from str_tbl where length(struct_element(c_struct, 'f3')) > 0",
+ "c_struct", false);
+ Assertions.assertEquals(
+ new TreeSet<>(ImmutableList.of(path("c_struct", "f3", "OFFSET"))),
+ new TreeSet<>(nestedNormalSlot.getAllAccessPaths().get()));
+ Assertions.assertEquals(
+ new TreeSet<>(ImmutableList.of(path("c_struct", "f3", "OFFSET"))),
+ new TreeSet<>(nestedNormalSlot.getPredicateAccessPaths().get()));
+
+ SlotReference nestedFragmentSlot = rewriteAndFindScanSlot(
+ "select 1 from str_tbl where length(struct_element(c_struct, 'f3')) > 0",
+ "c_struct", true);
+ assertNoAccessPaths(nestedFragmentSlot);
+ }
+
+ /**
+ * Verify that a specific string-typed column in the rewritten LogicalOlapScan either has
+ * BigIntType (offset-only optimization applied) or retains its original string type (suppressed).
+ *
+ * @param sql query to analyze and rewrite
+ * @param columnName name of the string column to inspect
+ * @param expectOptimized true → expect BigIntType + access paths; false → expect string type
+ * @param expectAllPaths expected access paths when {@code expectOptimized} is true
+ */
+ private void assertStringColumn(String sql, String columnName,
+ boolean expectOptimized, List<ColumnAccessPath> expectAllPaths) {
+ Plan rewritePlan = PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .getCascadesContext()
+ .getRewritePlan();
+
+ LogicalOlapScan scan = rewritePlan.collect(LogicalOlapScan.class::isInstance)
+ .stream()
+ .map(p -> (LogicalOlapScan) p)
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("No LogicalOlapScan in plan for: " + sql));
+
+ for (Slot slot : scan.getOutput()) {
+ if (!slot.getName().equalsIgnoreCase(columnName)) {
+ continue;
+ }
+ SlotReference slotRef = (SlotReference) slot;
+ if (expectOptimized) {
+ Optional<List<ColumnAccessPath>> allPaths = slotRef.getAllAccessPaths();
+ Assertions.assertTrue(allPaths.isPresent() && !allPaths.get().isEmpty(),
+ "Slot '" + columnName + "' should have access paths set");
+ Assertions.assertEquals(
+ new TreeSet<>(expectAllPaths),
+ new TreeSet<>(allPaths.get()),
+ "Unexpected access paths for slot '" + columnName + "'");
+ } else {
+ Assertions.assertNotEquals(BigIntType.INSTANCE, slotRef.getDataType(),
+ "Slot '" + columnName + "' should NOT be BigIntType (optimization suppressed)");
+ }
+ return;
+ }
+ Assertions.fail("Column '" + columnName + "' not found in LogicalOlapScan output for: " + sql);
+ }
+
+ private SlotReference rewriteAndFindScanSlot(String sql, String columnName,
+ boolean materializedViewRewritePlanFragment) {
+ PlanChecker planChecker = PlanChecker.from(connectContext).analyze(sql);
+ planChecker.getCascadesContext().setMaterializedViewRewritePlanFragment(materializedViewRewritePlanFragment);
+ Plan rewritePlan = planChecker.rewrite().getCascadesContext().getRewritePlan();
+ LogicalOlapScan scan = rewritePlan.collect(LogicalOlapScan.class::isInstance)
+ .stream()
+ .map(p -> (LogicalOlapScan) p)
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("No LogicalOlapScan in plan for: " + sql));
+ return scan.getOutput().stream()
+ .filter(slot -> slot.getName().equalsIgnoreCase(columnName))
+ .map(slot -> (SlotReference) slot)
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("Column '" + columnName
+ + "' not found in LogicalOlapScan output for: " + sql));
+ }
+
+ private void assertNoAccessPaths(SlotReference slot) {
+ Assertions.assertTrue(!slot.getAllAccessPaths().isPresent() || slot.getAllAccessPaths().get().isEmpty());
+ Assertions.assertTrue(!slot.getPredicateAccessPaths().isPresent()
+ || slot.getPredicateAccessPaths().get().isEmpty());
+ }
private Pair<PhysicalPlan, List<SlotDescriptor>> collectComplexSlots(String sql) throws Exception {
NereidsPlanner planner = (NereidsPlanner) executeNereidsSql(sql).planner();
diff --git a/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out b/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out
new file mode 100644
index 00000000000..440b393001c
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out
@@ -0,0 +1,101 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !1 --
+
+-- !2 --
+
+-- !3 --
+1
+
+-- !4 --
+1
+
+-- !5 --
+0
+
+-- !6 --
+
+-- !7 --
+
+-- !8 --
+
+-- !9 --
+1
+
+-- !10 --
+
+-- !11 --
+
+-- !parent_null_with_child_data --
+10001
+
+-- !12 --
+
+-- !13 --
+\N
+
+-- !14 --
+0
+
+-- !15 --
+1
+
+-- !16 --
+0
+
+-- !17 --
+1
+
+-- !map_values_parent_null_semantics --
+1
+
+-- !map_values_parent_not_null_semantics --
+2
+3
+
+-- !18 --
+0
+
+-- !19 --
+1
+
+-- !20 --
+0
+
+-- !21 --
+1
+
+-- !22 --
+1
+
+-- !23 --
+0
+
+-- !24 --
+0
+
+-- !25 --
+
+-- !26 --
+
+-- !27 --
+0
+
+-- !28 --
+0
+
+-- !29 --
+0
+
+-- !30 --
+0
+
+-- !31 --
+0
+
+-- !32 --
+0
+
+-- !33 --
+
+-- !34 --
+
diff --git a/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out b/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out
index 874bfea0ea0..439a81599ce 100644
--- a/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out
+++ b/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out
@@ -4,8 +4,20 @@
2 o mi
4 o yy
+-- !query2_0_before --
+1 \N yy
+1 o mm
+2 o mi
+4 o yy
+
-- !query1_0_after --
1 o mm
2 o mi
4 o yy
+-- !query2_0_after --
+1 \N yy
+1 o mm
+2 o mi
+4 o yy
+
diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy
new file mode 100644
index 00000000000..2707670f7d8
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy
@@ -0,0 +1,493 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Regression tests for the IS NULL / IS NOT NULL column pruning optimization.
+//
+// When IS NULL (or IS NOT NULL) is the *only* use of a nullable column, the FE
+// should emit a DATA access path with a "NULL" component so that the BE can
+// satisfy the query by reading only the null flag instead of the full column data.
+// The EXPLAIN plan should show:
+// nested columns: <col>: all access paths: [<col>.NULL]
+//
+// When the same column is also accessed for data (e.g., projected or used in
+// struct_element), the NULL-only path must be stripped from allAccessPaths but
+// preserved in predicateAccessPaths.
+
+suite("null_column_pruning") {
+ sql """ DROP TABLE IF EXISTS ncp_tbl """
+ sql """
+ CREATE TABLE ncp_tbl (
+ id INT,
+ str_col STRING NULL,
+ struct_col STRUCT<city: STRING, zip: INT> NULL,
+ arr_col ARRAY<INT> NULL,
+ map_col MAP<STRING, INT> NULL,
+ int_col INT NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+
+ sql """
+ INSERT INTO ncp_tbl VALUES
+ (1, 'hello', named_struct('city', null, 'zip', 10001), [1, 2, 3], {'a': 1, 'b': 2 }, 1)
+ """
+ // ─── Struct IS NULL only ────────────────────────────────────────────────────
+ // Only null check on struct_col → emit [struct_col, NULL] access path,
+ // type stays full struct (no pruning needed).
+ explain {
+ sql "select 1 from ncp_tbl where struct_col is null"
+ contains "nested columns"
+ contains "struct_col.NULL"
+ }
+
+ order_qt_1 "select 1 from ncp_tbl where struct_col is null";
+
+ // ─── String IS NULL only ────────────────────────────────────────────────────
+ // Only null check on str_col → emit [str_col, NULL] access path,
+ // type stays string (no pruning needed).
+ explain {
+ sql "select 1 from ncp_tbl where str_col is null"
+ contains "nested columns"
+ contains "str_col.NULL"
+ }
+
+ order_qt_2 "select 1 from ncp_tbl where str_col is null";
+
+ // ─── String IS NOT NULL only ────────────────────────────────────────────────
+ explain {
+ sql "select 1 from ncp_tbl where str_col is not null"
+ contains "nested columns"
+ contains "str_col.NULL"
+ }
+
+ order_qt_3 "select 1 from ncp_tbl where str_col is not null";
+
+ // ─── Struct IS NOT NULL only ────────────────────────────────────────────────
+ // IS NOT NULL is the same optimization (only null flag needed).
+ explain {
+ sql "select 1 from ncp_tbl where struct_col is not null"
+ contains "nested columns"
+ contains "struct_col.NULL"
+ }
+
+ order_qt_4 "select 1 from ncp_tbl where struct_col is not null";
+
+ // ─── Struct IS NULL in aggregate ────────────────────────────────────────────
+ explain {
+ sql "select count(*) from ncp_tbl where struct_col is null"
+ contains "nested columns"
+ contains "struct_col.NULL"
+ }
+
+ order_qt_5 "select count(*) from ncp_tbl where struct_col is null";
+
+ // ─── Array IS NULL only ─────────────────────────────────────────────────────
+ explain {
+ sql "select 1 from ncp_tbl where arr_col is null"
+ contains "nested columns"
+ contains "arr_col.NULL"
+ }
+
+ order_qt_6 "select 1 from ncp_tbl where arr_col is null";
+
+ // ─── Map IS NULL only ───────────────────────────────────────────────────────
+ explain {
+ sql "select 1 from ncp_tbl where map_col is null"
+ contains "nested columns"
+ contains "map_col.NULL"
+ }
+
+ order_qt_7 "select 1 from ncp_tbl where map_col is null";
+
+ // ─── Int IS NULL only ───────────────────────────────────────────────────────
+ // Nullable primitive type (INT) accessed only via IS NULL → emit [int_col, NULL]
+ // access path so BE only reads the null flag.
+ explain {
+ sql "select 1 from ncp_tbl where int_col is null"
+ contains "nested columns"
+ contains "int_col.NULL"
+ }
+
+ order_qt_8 "select 1 from ncp_tbl where int_col is null";
+
+ // ─── Int IS NOT NULL only ───────────────────────────────────────────────────
+ explain {
+ sql "select 1 from ncp_tbl where int_col is not null"
+ contains "nested columns"
+ contains "int_col.NULL"
+ }
+
+ order_qt_9 "select 1 from ncp_tbl where int_col is not null";
+
+ // ─── Mixed: int IS NULL + projected ────────────────────────────────────────
+ // int_col IS NULL in WHERE + int_col in SELECT → data is also needed.
+ // [int_col, NULL] stripped from allAccessPaths because [int_col] (full data)
+ // covers the same prefix and inherently includes the null flag.
+ explain {
+ sql "select int_col from ncp_tbl where int_col is null"
+ contains "nested columns"
+ contains "all access paths: [int_col]"
+ contains "predicate access paths: [int_col.NULL]"
+ }
+
+ order_qt_10 "select int_col from ncp_tbl where int_col is null";
+
+ // ─── Mixed: struct IS NULL + partial field access ───────────────────────────
+ // struct_col IS NULL in WHERE + struct_element in SELECT → child data is also needed.
+ // The parent struct_col.NULL path must NOT stay in allAccessPaths with child paths.
+ // BE StructFileColumnIterator treats a leading NULL sub-path as NULL_MAP_ONLY; if
+ // allAccessPaths were [struct_col.NULL, struct_col.city], BE would skip the city
+ // child iterator and default-fill the projected value. predicateAccessPaths still
+ // keeps struct_col.NULL so the predicate requirement is visible, while the normal
+ // nullable struct read materializes the parent null map together with child data.
+ explain {
+ sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null"
+ contains "nested columns"
+ contains "all access paths: [struct_col.city]"
+ contains "predicate access paths: [struct_col.NULL]"
+ }
+
+ order_qt_11 "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null";
+
+ // This query verifies the real correctness risk: one branch needs the parent null
+ // map, another branch needs a child null map, and the projection needs another
+ // child data path. Keeping struct_col.NULL in allAccessPaths would put BE in
+ // NULL_MAP_ONLY mode for the whole struct and return the default zip value instead
+ // of reading the zip child column.
+ explain {
+ sql "select struct_element(struct_col, 'zip') from ncp_tbl where struct_col is null or struct_element(struct_col, 'city') is null"
+ contains "nested columns"
+ contains "all access paths: [struct_col.city.NULL, struct_col.zip]"
+ contains "predicate access paths: [struct_col.NULL, struct_col.city.NULL]"
+ }
+
+ order_qt_parent_null_with_child_data "select struct_element(struct_col, 'zip') from ncp_tbl where struct_col is null or struct_element(struct_col, 'city') is null";
+
+ // ─── Non-optimizable: struct IS NULL + full struct projected ────────────────
+ // Full struct access covers its own null flag, so [struct_col.NULL] is stripped
+ // from allAccessPaths but kept in predicateAccessPaths.
+ explain {
+ sql "select struct_col from ncp_tbl where struct_col is null"
+ contains "nested columns"
+ contains "all access paths: [struct_col]"
+ contains "predicate access paths: [struct_col.NULL]"
+ }
+
+ order_qt_12 "select struct_col from ncp_tbl where struct_col is null";
+
+ // ─── Nested struct field IS NULL ────────────────────────────────────────────
+ // struct_element(struct_col, 'city') IS NULL should produce a null-flag-only
+ // predicate path [struct_col.city.NULL] while the projection reads city data.
+ // [struct_col.city.NULL] is stripped from allAccessPaths because [struct_col.city]
+ // covers the same prefix (full city data includes its null flag).
+ explain {
+ sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_element(struct_col, 'city') is null"
+ contains "nested columns"
+ contains "all access paths: [struct_col.city]"
+ contains "predicate access paths: [struct_col.city.NULL]"
+ }
+
+ order_qt_13 "select struct_element(struct_col, 'city') from ncp_tbl where struct_element(struct_col, 'city') is null";
+
+ // =========================================================================
+ // IS NULL on nested-type extraction functions (map_keys, map_values,
+ // element_at, struct_element, and nested combinations)
+ // =========================================================================
+
+ // ─── map_keys(map_col) IS NULL ─────────────────────────────────────────────
+ // map_keys(nullable_map) returns a NULL array only when the parent map itself
+ // is NULL, so the null-only path must be the parent map null map. Emitting
+ // map_col.KEYS.NULL would ask BE to inspect the key child null map instead.
+ explain {
+ sql "select count(1) from ncp_tbl where map_keys(map_col) is null"
+ contains "nested columns"
+ contains "map_col.NULL"
+ notContains "map_col.KEYS.NULL"
+ }
+
+ order_qt_14 "select count(1) from ncp_tbl where map_keys(map_col) is null";
+
+ // ─── map_keys(map_col) IS NOT NULL ──────────────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_tbl where map_keys(map_col) is not null"
+ contains "nested columns"
+ contains "map_col.NULL"
+ notContains "map_col.KEYS.NULL"
+ }
+
+ order_qt_15 "select count(1) from ncp_tbl where map_keys(map_col) is not null";
+
+ // ─── map_values(map_col) IS NULL ────────────────────────────────────────────
+ // A non-NULL map containing a NULL value, e.g. {'b': NULL}, still produces a
+ // non-NULL values array [NULL]. Therefore map_values(map_col) IS NULL is a
+ // parent-map null check, not a VALUES-child null check.
+ explain {
+ sql "select count(1) from ncp_tbl where map_values(map_col) is null"
+ contains "nested columns"
+ contains "map_col.NULL"
+ notContains "map_col.VALUES.NULL"
+ }
+
+ order_qt_16 "select count(1) from ncp_tbl where map_values(map_col) is null";
+
+ // ─── map_values(map_col) IS NOT NULL ────────────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_tbl where map_values(map_col) is not null"
+ contains "nested columns"
+ contains "map_col.NULL"
+ notContains "map_col.VALUES.NULL"
+ }
+
+ order_qt_17 "select count(1) from ncp_tbl where map_values(map_col) is not null";
+
+ sql """ DROP TABLE IF EXISTS ncp_map_null_semantics_tbl """
+ sql """
+ CREATE TABLE ncp_map_null_semantics_tbl (
+ id INT,
+ map_col MAP<STRING, INT> NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+ sql """
+ INSERT INTO ncp_map_null_semantics_tbl VALUES
+ (1, NULL),
+ (2, {'a': 1}),
+ (3, {'b': NULL})
+ """
+
+ explain {
+ sql "select id from ncp_map_null_semantics_tbl where map_values(map_col) is null order by id"
+ contains "nested columns"
+ contains "map_col.NULL"
+ notContains "map_col.VALUES.NULL"
+ }
+
+ // Only the NULL map row should match. Row 3 has a NULL value element, but
+ // map_values({'b': NULL}) is the non-NULL array [NULL].
+ order_qt_map_values_parent_null_semantics """
+ select id from ncp_map_null_semantics_tbl where map_values(map_col) is null order by id
+ """
+
+ order_qt_map_values_parent_not_null_semantics """
+ select id from ncp_map_null_semantics_tbl where map_values(map_col) is not null order by id
+ """
+
+ // ─── element_at(arr_col, 1) IS NULL ─────────────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_tbl where arr_col[1] is null"
+ contains "nested columns"
+ contains "arr_col.*.NULL"
+ }
+
+ order_qt_18 "select count(1) from ncp_tbl where arr_col[1] is null";
+
+ // ─── element_at(arr_col, 1) IS NOT NULL ─────────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_tbl where arr_col[1] is not null"
+ contains "nested columns"
+ contains "arr_col.*.NULL"
+ }
+
+ order_qt_19 "select count(1) from ncp_tbl where arr_col[1] is not null";
+
+ // ─── element_at(map_col, 'a') IS NULL ───────────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_tbl where map_col['a'] is null"
+ contains "nested columns"
+ contains "map_col.*.NULL"
+ }
+
+ order_qt_20 "select count(1) from ncp_tbl where map_col['a'] is null";
+
+ // ─── element_at(map_col, 'a') IS NOT NULL ───────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_tbl where map_col['a'] is not null"
+ contains "nested columns"
+ contains "map_col.*.NULL"
+ }
+
+ order_qt_21 "select count(1) from ncp_tbl where map_col['a'] is not null";
+
+ // ─── struct_element(struct_col, 'city') IS NULL only (no projection) ────────
+ explain {
+ sql "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is null"
+ contains "nested columns"
+ contains "struct_col.city.NULL"
+ }
+
+ order_qt_22 "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is null";
+
+ // ─── struct_element(struct_col, 'zip') IS NULL ──────────────────────────────
+ explain {
+ sql "select count(1) from ncp_tbl where struct_element(struct_col, 'zip') is null"
+ contains "nested columns"
+ contains "struct_col.zip.NULL"
+ }
+
+ order_qt_23 "select count(1) from ncp_tbl where struct_element(struct_col, 'zip') is null";
+
+ // ─── struct_element IS NOT NULL ─────────────────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is not null"
+ contains "nested columns"
+ contains "struct_col.city.NULL"
+ }
+
+ order_qt_24 "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is not null";
+
+ // ─── Mixed: map_keys IS NULL + map_keys projected ──────────────────────────
+ // Projection needs key data, while the predicate checks whether the parent map
+ // is NULL. The parent NULL path is kept only in predicateAccessPaths so BE does
+ // not switch the whole map iterator to NULL_MAP_ONLY and skip the keys child.
+ explain {
+ sql "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null"
+ contains "nested columns"
+ contains "all access paths: [map_col.KEYS]"
+ contains "predicate access paths: [map_col.NULL]"
+ }
+
+ order_qt_25 "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null";
+
+ // ─── Mixed: map_values IS NULL + map_values projected ──────────────────────
+ // Projection needs value data, while the predicate checks whether the parent
+ // map is NULL. A NULL value element does not make map_values(map_col) NULL.
+ explain {
+ sql "select map_values(map_col) from ncp_tbl where map_values(map_col) is null"
+ contains "nested columns"
+ contains "all access paths: [map_col.VALUES]"
+ contains "predicate access paths: [map_col.NULL]"
+ }
+
+ order_qt_26 "select map_values(map_col) from ncp_tbl where map_values(map_col) is null";
+
+ // ─── Nested types: struct containing map and array ─────────────────────────
+ sql """ DROP TABLE IF EXISTS ncp_nested_tbl """
+ sql """
+ CREATE TABLE ncp_nested_tbl (
+ id INT,
+ nested_struct STRUCT<inner_map: MAP<STRING, INT>, inner_arr: ARRAY<STRING>> NULL,
+ arr_of_structs ARRAY<STRUCT<name: STRING, age: INT>> NULL,
+ map_of_arrs MAP<STRING, ARRAY<INT>> NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+ sql """
+ INSERT INTO ncp_nested_tbl SELECT
+ 1,
+ named_struct('inner_map', map('x', 10), 'inner_arr', array('a', 'b')),
+ array(named_struct('name', 'Alice', 'age', 30)),
+ map('k', array(1, 2))
+ """
+
+ // ─── struct_element → map field IS NULL ─────────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_nested_tbl where struct_element(nested_struct, 'inner_map') is null"
+ contains "nested columns"
+ contains "nested_struct.inner_map.NULL"
+ }
+
+ order_qt_27 "select count(1) from ncp_nested_tbl where struct_element(nested_struct, 'inner_map') is null";
+
+ // ─── struct_element → array field IS NULL ───────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_nested_tbl where struct_element(nested_struct, 'inner_arr') is null"
+ contains "nested columns"
+ contains "nested_struct.inner_arr.NULL"
+ }
+
+ order_qt_28 "select count(1) from ncp_nested_tbl where struct_element(nested_struct, 'inner_arr') is null";
+
+ // ─── map_keys through struct_element IS NULL ────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_nested_tbl where map_keys(struct_element(nested_struct, 'inner_map')) is null"
+ contains "nested columns"
+ contains "nested_struct.inner_map.NULL"
+ notContains "nested_struct.inner_map.KEYS.NULL"
+ }
+
+ order_qt_29 "select count(1) from ncp_nested_tbl where map_keys(struct_element(nested_struct, 'inner_map')) is null";
+
+ // ─── map_values through struct_element IS NULL ──────────────────────────────
+ explain {
+ sql "select count(1) from ncp_nested_tbl where map_values(struct_element(nested_struct, 'inner_map')) is null"
+ contains "nested columns"
+ contains "nested_struct.inner_map.NULL"
+ notContains "nested_struct.inner_map.VALUES.NULL"
+ }
+
+ order_qt_30 "select count(1) from ncp_nested_tbl where map_values(struct_element(nested_struct, 'inner_map')) is null";
+
+ // ─── map_values(map_of_arrs) IS NULL ────────────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_nested_tbl where map_values(map_of_arrs) is null"
+ contains "nested columns"
+ contains "map_of_arrs.NULL"
+ notContains "map_of_arrs.VALUES.NULL"
+ }
+
+ order_qt_31 "select count(1) from ncp_nested_tbl where map_values(map_of_arrs) is null";
+
+ // ─── map_keys(map_of_arrs) IS NULL ──────────────────────────────────────────
+ explain {
+ sql "select count(1) from ncp_nested_tbl where map_keys(map_of_arrs) is null"
+ contains "nested columns"
+ contains "map_of_arrs.NULL"
+ notContains "map_of_arrs.KEYS.NULL"
+ }
+
+ order_qt_32 "select count(1) from ncp_nested_tbl where map_keys(map_of_arrs) is null";
+
+ // ─── Non-nullable column IS NULL → no nested column pruning ─────────────────
+ // A NOT NULL column has no null flags; IS NULL is always false and the optimizer
+ // must NOT generate a .NULL access path for it.
+ sql """ DROP TABLE IF EXISTS ncp_tbl_nn """
+ sql """
+ CREATE TABLE ncp_tbl_nn (
+ id INT NOT NULL,
+ str_col STRING NULL
+ ) ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+ sql """ INSERT INTO ncp_tbl_nn VALUES (1, 'hello') """
+
+ explain {
+ sql "select 1 from ncp_tbl_nn where id is null"
+ notContains "nested columns"
+ }
+
+ order_qt_33 "select 1 from ncp_tbl_nn where id is null";
+
+ // ─── length(str_col) = 0 OR str_col IS NULL ────────────────────────────────
+ // length(str_col) already uses the OFFSET path, and BE can derive null-ness
+ // from that layout, so the extra NULL-only path is redundant.
+ explain {
+ sql "select 1 from ncp_tbl where length(str_col) = 0 or str_col is null"
+ contains "nested columns"
+ contains "str_col.OFFSET"
+ notContains "str_col.NULL"
+ }
+
+ order_qt_34 "select 1 from ncp_tbl where length(str_col) = 0 or str_col is null";
+}
diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy
index 2fbc4717fe4..a0da286867c 100644
--- a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy
+++ b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy
@@ -32,6 +32,11 @@
// the plain string column should appear.
suite("string_length_column_pruning") {
+ // fe_debug performs strict nullability-change assertions in AdjustNullable.
+ // The IF/ORDER BY shape below can hit that pre-existing debug assertion before
+ // NestedColumnPruning runs, so keep this suite independent of global fe_debug.
+ sql "set fe_debug=false"
+
sql """ DROP TABLE IF EXISTS slcp_str_tbl """
sql """
CREATE TABLE slcp_str_tbl (
@@ -62,6 +67,18 @@ suite("string_length_column_pruning") {
notContains "type=bigint"
}
sql "select length(str_col) from slcp_str_tbl"
+
+ // length(str_col) in IF plus ORDER BY on a plain primitive column:
+ // only str_col should appear in nested columns, and NULL is redundant when OFFSET exists.
+ explain {
+ sql "select if(length(str_col) >= 5, true, false) a from slcp_str_tbl order by id"
+ contains "nested columns"
+ contains "str_col.OFFSET"
+ notContains "str_col.NULL"
+ notContains "all access paths: [id]"
+ }
+ sql "select if(length(str_col) >= 5, true, false) a from slcp_str_tbl order by id"
+
// Struct string field: length(struct_element) is the only use
explain {
sql "select length(struct_element(struct_col, 'f3')) from slcp_str_tbl"
diff --git a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy
index 16c6c4d1d1e..8c652503f8e 100644
--- a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy
@@ -16,7 +16,7 @@ package mv.unsafe_equals
// specific language governing permissions and limitations
// under the License.
-suite("null_unsafe_equals") {
+suite("null_un_safe_equals") {
String db = context.config.getDbNameByFile(context.file)
sql "use ${db}"
sql "set runtime_filter_mode=OFF";
@@ -75,8 +75,21 @@ suite("null_unsafe_equals") {
group by
o_orderstatus, o_comment;
"""
+ // query contains length(o_comment), which only needs string offsets in a standalone filter,
+ // but the final output still needs full o_comment data after mv rewrite.
+ def query2_0 =
+ """
+ select count(*), o_orderstatus, o_comment
+ from orders
+ where length(o_comment) > 0
+ group by
+ o_orderstatus, o_comment;
+ """
order_qt_query1_0_before "${query1_0}"
+ order_qt_query2_0_before "${query2_0}"
async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0")
+ mv_rewrite_success(query2_0, "mv1_0")
order_qt_query1_0_after "${query1_0}"
- sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0"""
+ order_qt_query2_0_after "${query2_0}"
+ // sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org