You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by en...@apache.org on 2026/03/17 05:05:51 UTC
(doris) branch master updated: [fix](lazy materialize) set uniqueId for lazy materialized slots (#61029)
This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c655b1a64d5 [fix](lazy materialize) set uniqueId for lazy materialized slots (#61029)
c655b1a64d5 is described below
commit c655b1a64d5e2fd21cbda2e28b6fcdd48b918873
Author: minghong <zh...@selectdb.com>
AuthorDate: Tue Mar 17 13:05:42 2026 +0800
[fix](lazy materialize) set uniqueId for lazy materialized slots (#61029)
This PR has two functional changes and one large documentation/renaming effort:
BE bug fix (materialization_opertor.cpp): Moves block_maps inside the per-relation loop to prevent stale entries when a backend returns a non-empty block for one relation but an empty block for another. This is a correct and important fix.
FE fix (PhysicalLazyMaterialize.java): Uses SlotReference.withColumn(originalColumn) so that createSlotDesc can write colUniqueId into the thrift SlotDescriptor. This is the core fix referenced by the PR title. The approach is sound — withColumn creates an immutable copy with the column set, which flows through createSlotDesc → setColumn → toThrift → setColUniqueId.
Renaming and documentation: lazyTableIdxs/idxs → lazyBaseColumnIndices/columnIdxsLists with extensive Javadoc. The naming is clearer.
---
be/src/exec/operator/materialization_opertor.cpp | 9 +++-
.../glue/translator/PhysicalPlanTranslator.java | 2 +-
.../plans/physical/PhysicalLazyMaterialize.java | 57 ++++++++++++++++++----
.../apache/doris/planner/MaterializationNode.java | 54 ++++++++++++++++++--
.../hive/test_hive_topn_lazy_mat.groovy | 6 +--
.../tvf/test_tvf_topn_lazy_mat.groovy | 4 +-
6 files changed, 109 insertions(+), 23 deletions(-)
diff --git a/be/src/exec/operator/materialization_opertor.cpp b/be/src/exec/operator/materialization_opertor.cpp
index 39c4fbcbb76..f75e445f6f5 100644
--- a/be/src/exec/operator/materialization_opertor.cpp
+++ b/be/src/exec/operator/materialization_opertor.cpp
@@ -54,9 +54,14 @@ void MaterializationSharedState::get_block(Block* block) {
}
Status MaterializationSharedState::merge_multi_response() {
- std::unordered_map<int64_t, std::pair<Block, int>> block_maps;
-
for (int i = 0; i < block_order_results.size(); ++i) {
+ // block_maps must be rebuilt for each relation (each i), because a backend that
+ // returned a non-empty block for relation i-1 may return an empty block for
+ // relation i (e.g. it holds rows only from one of the two tables in a UNION ALL).
+ // Keeping block_maps across iterations would leave stale entries from the previous
+ // relation and miss entries for the current one, causing the
+ // "backend_id not found in block_maps" error.
+ std::unordered_map<int64_t, std::pair<Block, int>> block_maps;
for (auto& [backend_id, rpc_struct] : rpc_struct_map) {
Block partial_block;
size_t uncompressed_size = 0;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 2c1518fdbaa..4100fab5c23 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -2779,7 +2779,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
materializeNode.setLazyColumns(materialize.getLazyColumns());
materializeNode.setLocations(materialize.getLazySlotLocations());
- materializeNode.setIdxs(materialize.getlazyTableIdxs());
+ materializeNode.setColumnIdxsLists(materialize.getLazyBaseColumnIndices());
List<Boolean> rowStoreFlags = new ArrayList<>();
for (Relation relation : materialize.getRelations()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java
index 5738a377823..25d39cc02b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java
@@ -62,11 +62,44 @@ public class PhysicalLazyMaterialize<CHILD_TYPE extends Plan> extends PhysicalUn
private final List<Slot> materializeInput;
private final List<Slot> materializeOutput;
- // used for BE
+ /**
+ * The following four fields are used by BE to perform the actual lazy fetch.
+ * They are indexed by relation: index i corresponds to relations.get(i).
+ *
+ * Example:
+ * SQL: SELECT t1.a, t1.b, t2.c, t2.d FROM t1 JOIN t2 ON ... WHERE t1.a > 5
+ * Assume t1.b and t2.d are lazily materialized (fetched after filtering).
+ *
+ * materializedSlots (non-lazy, computed eagerly) = [t1.a, t2.c]
+ * Output slot order = [t1.a(0), t2.c(1), t1.b(2), t2.d(3)]
+ *
+ * rowIdList = [row_id_t1, row_id_t2]
+ * The row-id slots passed to BE to locate the original rows.
+ *
+ * relations = [t1, t2]
+ *
+ * lazyColumns = [[Column(b)], [Column(d)]]
+ * For each relation, the Column objects to be lazily fetched.
+ *
+ * lazyBaseColumnIndices = [[colIdxOf(b) in t1], [colIdxOf(d) in t2]]
+ * For each relation, the physical column index inside the table
+ * for each lazy column (used by BE to locate the column on disk).
+ *
+ * lazySlotLocations = [[2], [3]]
+ * A two-level array: the outer level is indexed by relation
+ * (same as relations / rowIdList), and the inner level lists the
+ * output-tuple position for each lazy column of that relation.
+ * Two levels are needed because a single relation can have
+ * multiple lazy columns. For example, if both t1.b and t1.e
+ * were lazy, the entry for t1 would be [2, 4] (positions of b
+ * and e in the output tuple), while t2 remains [3].
+ * BE uses each position to know which output slot to fill in
+ * after fetching the column value from disk.
+ */
private final List<Slot> rowIdList;
private List<List<Column>> lazyColumns = new ArrayList<>();
private List<List<Integer>> lazySlotLocations = new ArrayList<>();
- private List<List<Integer>> lazyTableIdxs = new ArrayList<>();
+ private List<List<Integer>> lazyBaseColumnIndices = new ArrayList<>();
private final List<Relation> relations;
@@ -101,7 +134,7 @@ public class PhysicalLazyMaterialize<CHILD_TYPE extends Plan> extends PhysicalUn
this.materializedSlots = ImmutableList.copyOf(materializedSlots);
this.materializeMap = materializeMap;
lazySlotLocations = new ArrayList<>();
- lazyTableIdxs = new ArrayList<>();
+ lazyBaseColumnIndices = new ArrayList<>();
lazyColumns = new ArrayList<>();
ImmutableList.Builder<Slot> outputBuilder = ImmutableList.builder();
@@ -126,15 +159,19 @@ public class PhysicalLazyMaterialize<CHILD_TYPE extends Plan> extends PhysicalUn
List<Column> lazyColumnForRel = new ArrayList<>();
lazyColumns.add(lazyColumnForRel);
- List<Integer> lazyIdxForRel = new ArrayList<>();
- lazyTableIdxs.add(lazyIdxForRel);
+ List<Integer> lazyBaseColumnIdxForRel = new ArrayList<>();
+ lazyBaseColumnIndices.add(lazyBaseColumnIdxForRel);
List<Integer> lazySlotLocationForRel = new ArrayList<>();
lazySlotLocations.add(lazySlotLocationForRel);
for (Slot lazySlot : relationToLazySlotMap.get(rel)) {
- outputBuilder.add(lazySlot);
- lazyColumnForRel.add(materializeMap.get(lazySlot).baseSlot.getOriginalColumn().get());
- lazyIdxForRel.add(relationTable.getBaseColumnIdxByName(lazySlot.getName()));
+ // Set originalColumn on the lazy slot so that createSlotDesc can write
+ // colUniqueId into the thrift SlotDescriptor — BE needs it to resolve
+ // the column during remote fetch.
+ Column originalColumn = materializeMap.get(lazySlot).baseSlot.getOriginalColumn().get();
+ outputBuilder.add(((SlotReference) lazySlot).withColumn(originalColumn));
+ lazyColumnForRel.add(originalColumn);
+ lazyBaseColumnIdxForRel.add(relationTable.getBaseColumnIdxByName(lazySlot.getName()));
lazySlotLocationForRel.add(loc);
loc++;
}
@@ -250,8 +287,8 @@ public class PhysicalLazyMaterialize<CHILD_TYPE extends Plan> extends PhysicalUn
return lazySlotLocations;
}
- public List<List<Integer>> getlazyTableIdxs() {
- return lazyTableIdxs;
+ public List<List<Integer>> getLazyBaseColumnIndices() {
+ return lazyBaseColumnIndices;
}
public List<Slot> getRowIds() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
index 40d16a88faf..1acb473d851 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java
@@ -71,6 +71,50 @@ import java.util.List;
* }
*/
public class MaterializationNode extends PlanNode {
+ /**
+ * Example to illustrate the fields below:
+ *
+ * SQL: SELECT pk, col_date FROM t ORDER BY pk LIMIT 10
+ * (col_date is chosen for lazy materialization)
+ *
+ * The child plan produces: [pk(eager), row_id_t]
+ * row_id_t is a "global row-id" slot that encodes where the real col_date lives on disk.
+ *
+ * materializeTupleDescriptor (intermediate tuple, e.g. tuple 5):
+ * index 0 → pk (eager slot, already in the child block)
+ * index 1 → col_date (lazy slot, to be fetched remotely via row_id)
+ *
+ * outputTupleDesc (final output tuple, e.g. tuple 6):
+ * contains the columns actually returned to the parent node after
+ * applying projectList on top of materializeTupleDescriptor.
+ * In simple cases it is the same set of columns.
+ * Example: projectList = [pk#0, col_date#1] → outputTupleDesc = {pk, col_date}
+ *
+ * Relationship:
+ * materializeTupleDescriptor = "working" tuple that holds eager + lazy columns,
+ * used to build the RPC fetch request and place the
+ * fetched values back into the block.
+ * outputTupleDesc = "public" tuple visible to the parent; produced by
+ * projecting materializeTupleDescriptor through projectList.
+ *
+ * locations (= PhysicalLazyMaterialize.lazySlotLocations):
+ * For each relation, the index of each lazy slot inside materializeTupleDescriptor.
+ * In the example above: [[1]]
+ * → outer list index 0 = first (only) relation t
+ * → inner value 1 = col_date is at index 1 in materializeTupleDescriptor.slots()
+ * BE uses this to call slots[location]->to_protobuf(...) when building the fetch
+ * request so the remote side knows the schema of what to return.
+ *
+ * columnIdxsLists (= PhysicalLazyMaterialize.lazyBaseColumnIndices):
+ * For each relation, the physical column index in the table for each lazy column.
+ * In the example above: [[3]] (col_date is the 3rd column in t, 0-indexed)
+ * BE sends this index to the storage layer so it can locate the column on disk
+ * without needing the full schema.
+ *
+ * If two columns b and c from table t were both lazy, and the column indices in the
+ * table are 2 and 5 respectively, and they occupy slots at index 1 and 2 in the tuple:
+ * locations = [[1, 2]], columnIdxsLists = [[2, 5]]
+ */
private TPaloNodesInfo nodesInfo;
private TupleDescriptor materializeTupleDescriptor;
@@ -79,7 +123,7 @@ public class MaterializationNode extends PlanNode {
private List<List<Column>> lazyColumns;
private List<List<Integer>> locations;
- private List<List<Integer>> idxs;
+ private List<List<Integer>> columnIdxsLists;
private List<Boolean> rowStoreFlags;
@@ -124,7 +168,7 @@ public class MaterializationNode extends PlanNode {
}
output.append(detailPrefix).append("column_descs_lists").append(lazyColumns).append("\n");
output.append(detailPrefix).append("locations: ").append(locations).append("\n");
- output.append(detailPrefix).append("table_idxs: ").append(idxs).append("\n");
+ output.append(detailPrefix).append("column_idxs_lists: ").append(columnIdxsLists).append("\n");
output.append(detailPrefix).append("row_ids: ").append(rowIds).append("\n");
output.append(detailPrefix).append("isTopMaterializeNode: ").append(isTopMaterializeNode).append("\n");
printNestedColumns(output, detailPrefix, outputTupleDesc);
@@ -152,7 +196,7 @@ public class MaterializationNode extends PlanNode {
msg.materialization_node.setColumnDescsLists(thriftCols);
msg.materialization_node.setSlotLocsLists(locations);
- msg.materialization_node.setColumnIdxsLists(idxs);
+ msg.materialization_node.setColumnIdxsLists(columnIdxsLists);
msg.materialization_node.setFetchRowStores(rowStoreFlags);
msg.materialization_node.setGcIdMap(isTopMaterializeNode);
}
@@ -169,8 +213,8 @@ public class MaterializationNode extends PlanNode {
this.locations = locations;
}
- public void setIdxs(List<List<Integer>> idxs) {
- this.idxs = idxs;
+ public void setColumnIdxsLists(List<List<Integer>> columnIdxsLists) {
+ this.columnIdxsLists = columnIdxsLists;
}
public void setRowStoreFlags(List<Boolean> rowStoreFlags) {
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
index e90ab9b8631..761cd1cddf9 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy
@@ -210,7 +210,7 @@ suite("test_hive_topn_lazy_mat", "p0,external") {
contains("projectList:[id, name, value, active, score, file_id]")
contains("column_descs_lists[[`name` text NULL, `value` double NULL, `active` boolean NULL, `score` double NULL, `file_id` int NULL]]")
contains("locations: [[1, 2, 3, 4, 5]]")
- contains("table_idxs: [[1, 2, 3, 4, 5]]")
+ contains("column_idxs_lists: [[1, 2, 3, 4, 5]]")
contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]")
}
@@ -219,7 +219,7 @@ suite("test_hive_topn_lazy_mat", "p0,external") {
contains("projectList:[file_id, id]")
contains("column_descs_lists[[`id` int NULL, `file_id` int NULL]]")
contains("locations: [[1, 2]]")
- contains("table_idxs: [[0, 5]]")
+ contains("column_idxs_lists: [[0, 5]]")
contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]")
}
@@ -229,7 +229,7 @@ suite("test_hive_topn_lazy_mat", "p0,external") {
contains("projectList:[name, length(a.name), value, id, name, value, active, score, file_id, id, name, value, active, score, file_id]")
contains("column_descs_lists[[`name` text NULL, `value` double NULL, `active` boolean NULL, `score` double NULL, `file_id` int NULL], [`value` double NULL, `active` boolean NULL, `score` double NULL, `file_id` int NULL]]")
contains("locations: [[5, 6, 7, 8, 9], [10, 11, 12, 13]]")
- contains("table_idxs: [[1, 2, 3, 4, 5], [2, 3, 4, 5]]")
+ contains("column_idxs_lists: [[1, 2, 3, 4, 5], [2, 3, 4, 5]]")
contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table, __DORIS_GLOBAL_ROWID_COL__parquet_topn_lazy_mat_table]")
}
diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
index b1db8faa943..d084631c772 100644
--- a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy
@@ -148,7 +148,7 @@ suite("test_tvf_topn_lazy_mat", "p0,external") {
contains("column_descs_lists[[`name` text NULL, `value` double NULL, `active` boolean NULL, `score` double NULL]]")
contains("locations: [[1, 2, 3, 4]]")
- contains("table_idxs: [[1, 2, 3, 4]]")
+ contains("column_idxs_lists: [[1, 2, 3, 4]]")
contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__hdfs]")
contains("isTopMaterializeNode: true")
contains("SlotDescriptor{id=0, col=id, colUniqueId=-1, type=bigint, nullable=true")
@@ -163,7 +163,7 @@ suite("test_tvf_topn_lazy_mat", "p0,external") {
contains("projectList:[name, value, score]")
contains("column_descs_lists[[`name` text NULL, `value` double NULL, `score` double NULL]]")
contains("locations: [[1, 2, 3]]")
- contains("table_idxs: [[1, 2, 4]]")
+ contains("column_idxs_lists: [[1, 2, 4]]")
contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__hdfs]")
contains("isTopMaterializeNode: true")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org