You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2025/11/12 15:42:28 UTC
(doris) branch master updated: [chore](plan) remove ColumnStats and mem layout on tuple (#57912)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 71d706ed653 [chore](plan) remove ColumnStats and mem layout on tuple (#57912)
71d706ed653 is described below
commit 71d706ed6531d6ed2c974ae9d1c1bc6061a57f01
Author: morrySnow <zh...@selectdb.com>
AuthorDate: Wed Nov 12 23:42:18 2025 +0800
[chore](plan) remove ColumnStats and mem layout on tuple (#57912)
---
.../org/apache/doris/catalog/PrimitiveType.java | 5 -
.../java/org/apache/doris/alter/RollupJobV2.java | 1 -
.../org/apache/doris/alter/SchemaChangeJobV2.java | 1 -
.../org/apache/doris/analysis/AggregateInfo.java | 4 +-
.../org/apache/doris/analysis/DescriptorTable.java | 150 +-----------
.../org/apache/doris/analysis/SlotDescriptor.java | 110 +--------
.../org/apache/doris/analysis/TupleDescriptor.java | 262 +--------------------
.../java/org/apache/doris/catalog/ColumnStats.java | 134 -----------
.../apache/doris/datasource/FileQueryScanNode.java | 7 -
.../doris/source/RemoteDorisScanNode.java | 3 -
.../doris/datasource/jdbc/source/JdbcScanNode.java | 3 -
.../doris/datasource/odbc/source/OdbcScanNode.java | 3 -
.../source/TrinoConnectorScanNode.java | 3 -
.../datasource/tvf/source/MetadataScanNode.java | 1 -
.../glue/translator/PhysicalPlanTranslator.java | 52 +---
.../glue/translator/PlanTranslatorContext.java | 1 -
.../org/apache/doris/planner/OlapScanNode.java | 7 -
.../org/apache/doris/planner/ResultFileSink.java | 3 -
.../org/apache/doris/catalog/ColumnStatTest.java | 87 -------
.../translator/PhysicalPlanTranslatorTest.java | 2 +-
20 files changed, 26 insertions(+), 813 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
index 71966b8c7ac..4b6c47a2587 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
@@ -374,11 +374,6 @@ public enum PrimitiveType {
return (this == VARCHAR || this == CHAR || this == STRING);
}
- public boolean isIntegerType() {
- return (this == TINYINT || this == SMALLINT
- || this == INT || this == BIGINT);
- }
-
public boolean isIPType() {
return (this == IPV4 || this == IPV6);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 7701d04753d..2cbb21489b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -452,7 +452,6 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
for (Column column : rollupColumns) {
SlotDescriptor destSlotDesc = descTable.addSlotDescriptor(destTupleDesc);
- destSlotDesc.setIsMaterialized(true);
destSlotDesc.setColumn(column);
destSlotDesc.setIsNullable(column.isAllowNull());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 48e5617604f..0ae7a0c518d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -526,7 +526,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 implements GsonPostProcessable
TupleDescriptor destTupleDesc = descTable.createTupleDescriptor();
for (Column column : fullSchema) {
SlotDescriptor destSlotDesc = descTable.addSlotDescriptor(destTupleDesc);
- destSlotDesc.setIsMaterialized(true);
destSlotDesc.setColumn(column);
destSlotDesc.setIsNullable(column.isAllowNull());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java
index 934a109ef64..09c9b705974 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java
@@ -105,9 +105,7 @@ public final class AggregateInfo extends AggregateInfoBase {
int groupingExprNum = groupingExprs != null ? groupingExprs.size() : 0;
Preconditions.checkState(groupingExprNum <= outputSlots.size());
for (int i = groupingExprNum; i < outputSlots.size(); ++i) {
- if (outputSlots.get(i).isMaterialized()) {
- materializedSlots.add(i - groupingExprNum);
- }
+ materializedSlots.add(i - groupingExprNum);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java
index fb6cc7df0a8..254edea5fbb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java
@@ -21,19 +21,12 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.thrift.TDescriptorTable;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
/**
@@ -42,18 +35,10 @@ import java.util.Map;
* them unique ids..
*/
public class DescriptorTable {
- private static final Logger LOG = LogManager.getLogger(DescriptorTable.class);
-
private final HashMap<TupleId, TupleDescriptor> tupleDescs = new HashMap<TupleId, TupleDescriptor>();
- // List of referenced tables with no associated TupleDescriptor to ship to the BE.
- // For example, the output table of an insert query.
- private final List<TableIf> referencedTables = new ArrayList<TableIf>();
private final IdGenerator<TupleId> tupleIdGenerator = TupleId.createGenerator();
private final IdGenerator<SlotId> slotIdGenerator = SlotId.createGenerator();
private final HashMap<SlotId, SlotDescriptor> slotDescs = Maps.newHashMap();
-
- private final HashMap<SlotDescriptor, SlotDescriptor> outToIntermediateSlots = new HashMap<>();
-
private TDescriptorTable thriftDescTable = null; // serialized version of this
public DescriptorTable() {
@@ -78,32 +63,6 @@ public class DescriptorTable {
return result;
}
- /**
- * Used by new optimizer.
- */
- public SlotDescriptor addSlotDescriptor(TupleDescriptor d, int id) {
- SlotDescriptor result = new SlotDescriptor(new SlotId(id), d);
- d.addSlot(result);
- slotDescs.put(result.getId(), result);
- return result;
- }
-
- /**
- * Create copy of src with new id. The returned descriptor has its mem layout
- * computed.
- */
- public TupleDescriptor copyTupleDescriptor(TupleId srcId, String debugName) {
- TupleDescriptor d = new TupleDescriptor(tupleIdGenerator.getNextId(), debugName);
- tupleDescs.put(d.getId(), d);
- // create copies of slots
- TupleDescriptor src = tupleDescs.get(srcId);
- for (SlotDescriptor slot : src.getSlots()) {
- copySlotDescriptor(d, slot);
- }
- d.computeStatAndMemLayout();
- return d;
- }
-
/**
* Append copy of src to dest.
*/
@@ -118,71 +77,6 @@ public class DescriptorTable {
return tupleDescs.get(id);
}
- public HashMap<SlotId, SlotDescriptor> getSlotDescs() {
- return slotDescs;
- }
-
- /**
- * Return all tuple desc by idList.
- */
- public List<TupleDescriptor> getTupleDesc(List<TupleId> idList) throws AnalysisException {
- List<TupleDescriptor> result = Lists.newArrayList();
- for (TupleId tupleId : idList) {
- TupleDescriptor tupleDescriptor = getTupleDesc(tupleId);
- if (tupleDescriptor == null) {
- throw new AnalysisException("Invalid tuple id:" + tupleId.toString());
- }
- result.add(tupleDescriptor);
- }
- return result;
- }
-
- public SlotDescriptor getSlotDesc(SlotId id) {
- return slotDescs.get(id);
- }
-
- public Collection<TupleDescriptor> getTupleDescs() {
- return tupleDescs.values();
- }
-
- public void addReferencedTable(TableIf table) {
- referencedTables.add(table);
- }
-
- /**
- * Marks all slots in list as materialized.
- */
- public void markSlotsMaterialized(List<SlotId> ids) {
- for (SlotId id : ids) {
- getSlotDesc(id).setIsMaterialized(true);
- }
- }
-
- @Deprecated
- public void computeMemLayout() {
- for (TupleDescriptor d : tupleDescs.values()) {
- d.computeMemLayout();
- }
- }
-
- // Computes physical layout parameters of all descriptors and calculate the statistics of the tuple.
- // Call this only after the last descriptor was added.
- public void computeStatAndMemLayout() {
- for (TupleDescriptor d : tupleDescs.values()) {
- d.computeStatAndMemLayout();
- }
- }
-
- public void addSlotMappingInfo(Map<SlotDescriptor, SlotDescriptor> mapping) {
- outToIntermediateSlots.putAll(mapping);
- }
-
- public void materializeIntermediateSlots() {
- for (Map.Entry<SlotDescriptor, SlotDescriptor> entry : outToIntermediateSlots.entrySet()) {
- entry.getValue().setIsMaterialized(entry.getKey().isMaterialized());
- }
- }
-
public TDescriptorTable toThrift() {
if (thriftDescTable != null) {
return thriftDescTable;
@@ -193,22 +87,16 @@ public class DescriptorTable {
for (TupleDescriptor tupleD : tupleDescs.values()) {
// inline view of a non-constant select has a non-materialized tuple descriptor
// in the descriptor table just for type checking, which we need to skip
- if (tupleD.isMaterialized()) {
- result.addToTupleDescriptors(tupleD.toThrift());
- // an inline view of a constant select has a materialized tuple
- // but its table has no id
- if (tupleD.getTable() != null
- && tupleD.getTable().getId() >= 0) {
- referencedTbls.put(tupleD.getTable().getId(), tupleD.getTable());
- }
- for (SlotDescriptor slotD : tupleD.getMaterializedSlots()) {
- result.addToSlotDescriptors(slotD.toThrift());
- }
+ result.addToTupleDescriptors(tupleD.toThrift());
+ // an inline view of a constant select has a materialized tuple
+ // but its table has no id
+ if (tupleD.getTable() != null
+ && tupleD.getTable().getId() >= 0) {
+ referencedTbls.put(tupleD.getTable().getId(), tupleD.getTable());
+ }
+ for (SlotDescriptor slotD : tupleD.getSlots()) {
+ result.addToSlotDescriptors(slotD.toThrift());
}
- }
-
- for (TableIf tbl : referencedTables) {
- referencedTbls.put(tbl.getId(), tbl);
}
for (TableIf tbl : referencedTbls.values()) {
@@ -218,29 +106,11 @@ public class DescriptorTable {
return result;
}
- public String debugString() {
- StringBuilder out = new StringBuilder();
- out.append("tuples:\n");
- for (TupleDescriptor desc : tupleDescs.values()) {
- out.append(desc).append("\n");
- }
- out.append("\n ");
- out.append("slotDesc size: ").append(slotDescs.size()).append("\n");
- for (SlotDescriptor desc : slotDescs.values()) {
- out.append(desc.debugString());
- out.append("\n");
- }
- out.append("\n ");
- return out.toString();
- }
-
public String getExplainString() {
StringBuilder out = new StringBuilder();
out.append("\nTuples:\n");
for (TupleDescriptor desc : tupleDescs.values()) {
- if (desc.isMaterialized()) {
- out.append(desc.getExplainString()).append("\n");
- }
+ out.append(desc.getExplainString()).append("\n");
}
return out.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
index ca8b1a7f72f..d4376f5eaf1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
@@ -21,7 +21,6 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.ColumnStats;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Type;
import org.apache.doris.thrift.TSlotDescriptor;
@@ -57,21 +56,9 @@ public class SlotDescriptor {
// path_ is set.
private List<Expr> sourceExprs = Lists.newArrayList();
- // if false, this slot doesn't need to be materialized in parent tuple
- // (and physical layout parameters are invalid)
- private boolean isMaterialized;
-
// if false, this slot cannot be NULL
private boolean isNullable;
- // physical layout parameters
- private int byteSize;
- private int byteOffset = 0; // within tuple
- private int slotIdx; // index within tuple struct
- private int slotOffset; // index within slot array list
-
- private ColumnStats stats; // only set if 'column' isn't set
- private boolean isAgg;
// If set to false, then such slots will be ignored during
// materialize them.Used to optimize to read less data and less memory usage
private boolean needMaterialize = true;
@@ -82,31 +69,18 @@ public class SlotDescriptor {
this.id = id;
this.parent = parent;
- this.byteOffset = -1; // invalid
- this.isMaterialized = false;
this.isNullable = true;
- this.isAgg = false;
}
public SlotDescriptor(SlotId id, TupleDescriptor parent, SlotDescriptor src) {
this.id = id;
this.parent = parent;
- this.byteOffset = src.byteOffset;
- this.slotIdx = src.slotIdx;
- this.isMaterialized = src.isMaterialized;
this.column = src.column;
this.isNullable = src.isNullable;
- this.byteSize = src.byteSize;
- this.isAgg = false;
- this.stats = src.stats;
this.type = src.type;
this.sourceExprs.add(new SlotRef(src));
}
- public boolean getIsAgg() {
- return isAgg;
- }
-
public void setNeedMaterialize(boolean needMaterialize) {
this.needMaterialize = needMaterialize;
}
@@ -115,10 +89,6 @@ public class SlotDescriptor {
return !this.needMaterialize;
}
- public void setIsAgg(boolean agg) {
- isAgg = agg;
- }
-
public SlotId getId() {
return id;
}
@@ -157,14 +127,6 @@ public class SlotDescriptor {
this.column = column;
}
- public boolean isMaterialized() {
- return isMaterialized;
- }
-
- public void setIsMaterialized(boolean value) {
- isMaterialized = value;
- }
-
public boolean isAutoInc() {
return isAutoInc;
}
@@ -181,51 +143,10 @@ public class SlotDescriptor {
isNullable = value;
}
- public int getByteSize() {
- return byteSize;
- }
-
- public void setByteSize(int byteSize) {
- this.byteSize = byteSize;
- }
-
- public int getByteOffset() {
- return byteOffset;
- }
-
- public void setByteOffset(int byteOffset) {
- this.byteOffset = byteOffset;
- }
-
- public void setSlotIdx(int slotIdx) {
- this.slotIdx = slotIdx;
- }
-
- public void setStats(ColumnStats stats) {
- this.stats = stats;
- }
-
public void setMaterializedColumnName(String name) {
this.materializedColumnName = name;
}
- public ColumnStats getStats() {
- if (stats == null) {
- stats = new ColumnStats();
- }
- // FIXME(dhc): mock ndv
- stats.setNumDistinctValues((long) parent.getCardinality());
- return stats;
- }
-
- public void setSlotOffset(int slotOffset) {
- this.slotOffset = slotOffset;
- }
-
- public int getSlotOffset() {
- return slotOffset;
- }
-
public String getLabel() {
return label;
}
@@ -238,10 +159,6 @@ public class SlotDescriptor {
sourceExprs = Collections.singletonList(expr);
}
- public void addSourceExpr(Expr expr) {
- sourceExprs.add(expr);
- }
-
public List<Expr> getSourceExprs() {
return sourceExprs;
}
@@ -261,33 +178,13 @@ public class SlotDescriptor {
this.virtualColumn = virtualColumn;
}
- /**
- * Return true if the physical layout of this descriptor matches the physical layout
- * of the other descriptor, but not necessarily ids.
- */
- public boolean layoutEquals(SlotDescriptor other) {
- if (!getType().equals(other.getType())) {
- return false;
- }
- if (isNullable != other.isNullable) {
- return false;
- }
- if (getByteSize() != other.getByteSize()) {
- return false;
- }
- if (getByteOffset() != other.getByteOffset()) {
- return false;
- }
- return true;
- }
-
public TSlotDescriptor toThrift() {
// Non-nullable slots will have 0 for the byte offset and -1 for the bit mask
String colName = materializedColumnName != null ? materializedColumnName :
((column != null) ? column.getNonShadowName() : "");
TSlotDescriptor tSlotDescriptor = new TSlotDescriptor(id.asInt(), parent.getId().asInt(), type.toThrift(), -1,
- byteOffset, 0, getIsNullable() ? 0 : -1, colName, slotIdx,
- isMaterialized);
+ 0, 0, getIsNullable() ? 0 : -1, colName, -1,
+ true);
tSlotDescriptor.setNeedMaterialize(needMaterialize);
tSlotDescriptor.setIsAutoIncrement(isAutoInc);
if (column != null) {
@@ -344,8 +241,7 @@ public class SlotDescriptor {
String typeStr = (type == null ? "null" : type.toString());
String parentTupleId = (parent == null) ? "null" : parent.getId().toString();
return MoreObjects.toStringHelper(this).add("id", id.asInt()).add("parent", parentTupleId).add("col", caption)
- .add("type", typeStr).add("materialized", isMaterialized).add("byteSize", byteSize)
- .add("byteOffset", byteOffset).add("slotIdx", slotIdx).add("nullable", getIsNullable())
+ .add("type", typeStr).add("nullable", getIsNullable())
.add("isAutoIncrement", isAutoInc).add("subColPath", subColPath)
.add("virtualColumn", virtualColumn == null ? null : virtualColumn.toSql()).toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java
index 905cc9bf42d..8a3bec64987 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java
@@ -20,30 +20,18 @@
package org.apache.doris.analysis;
-import org.apache.doris.catalog.ColumnStats;
-import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.info.TableNameInfo;
import org.apache.doris.info.TableRefInfo;
import org.apache.doris.thrift.TTupleDescriptor;
import com.google.common.base.Joiner;
import com.google.common.base.MoreObjects;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
public class TupleDescriptor {
- private static final Logger LOG = LogManager.getLogger(TupleDescriptor.class);
private final TupleId id;
private final String debugName; // debug only
private final ArrayList<SlotDescriptor> slots;
@@ -53,43 +41,21 @@ public class TupleDescriptor {
// underlying table, if there is one
private TableRefInfo ref;
- // All legal aliases of this tuple.
- private String[] aliases;
-
- // If true, requires that aliases_.length() == 1. However, aliases_.length() == 1
- // does not imply an explicit alias because nested collection refs have only a
- // single implicit alias.
- private boolean hasExplicitAlias;
-
- // if false, this tuple doesn't need to be materialized
- private boolean isMaterialized = true;
-
- private int byteSize; // of all slots plus null indicators
-
- // This cardinality is only used to mock slot ndv.
- // Only tuple of olap scan node has this value.
- private long cardinality;
-
- private float avgSerializedSize; // in bytes; includes serialization overhead
-
private int tableId = -1;
public TupleDescriptor(TupleId id) {
this.id = id;
this.slots = new ArrayList<SlotDescriptor>();
this.debugName = "";
- this.cardinality = -1;
}
public TupleDescriptor(TupleId id, String debugName) {
this.id = id;
this.slots = new ArrayList<SlotDescriptor>();
this.debugName = debugName;
- this.cardinality = -1;
}
public void addSlot(SlotDescriptor desc) {
- desc.setSlotOffset(slots.size());
slots.add(desc);
}
@@ -109,10 +75,6 @@ public class TupleDescriptor {
return slots;
}
- public void setTableId(int id) {
- tableId = id;
- }
-
/**
* get slot desc by slot id.
*
@@ -128,34 +90,6 @@ public class TupleDescriptor {
return null;
}
- public long getCardinality() {
- return cardinality;
- }
-
- public void setCardinality(long cardinality) {
- this.cardinality = cardinality;
- }
-
- public ArrayList<SlotDescriptor> getMaterializedSlots() {
- ArrayList<SlotDescriptor> result = Lists.newArrayList();
- for (SlotDescriptor slot : slots) {
- if (slot.isMaterialized()) {
- result.add(slot);
- }
- }
- return result;
- }
-
- public ArrayList<SlotId> getMaterializedSlotIds() {
- ArrayList<SlotId> result = Lists.newArrayList();
- for (SlotDescriptor slot : slots) {
- if (slot.isMaterialized()) {
- result.add(slot.getId());
- }
- }
- return result;
- }
-
public ArrayList<SlotId> getAllSlotIds() {
ArrayList<SlotId> result = Lists.newArrayList();
for (SlotDescriptor slot : slots) {
@@ -177,15 +111,6 @@ public class TupleDescriptor {
return null;
}
- public boolean hasVariantCol() {
- for (SlotDescriptor slotDesc : slots) {
- if (slotDesc.getColumn() != null && slotDesc.getColumn().getType().isVariantType()) {
- return true;
- }
- }
- return false;
- }
-
public TableIf getTable() {
return table;
}
@@ -194,43 +119,6 @@ public class TupleDescriptor {
table = tbl;
}
- public int getByteSize() {
- return byteSize;
- }
-
- public void setIsMaterialized(boolean value) {
- isMaterialized = value;
- }
-
- public boolean isMaterialized() {
- return isMaterialized;
- }
-
- public float getAvgSerializedSize() {
- return avgSerializedSize;
- }
-
- public void setAliases(String[] aliases, boolean hasExplicitAlias) {
- this.aliases = aliases;
- this.hasExplicitAlias = hasExplicitAlias;
- }
-
- public boolean hasExplicitAlias() {
- return hasExplicitAlias;
- }
-
- public String getAlias() {
- return (aliases != null) ? aliases[0] : null;
- }
-
- public String getLastAlias() {
- return (aliases != null) ? aliases[aliases.length - 1] : null;
- }
-
- public TableNameInfo getAliasAsName() {
- return (aliases != null) ? new TableNameInfo(aliases[0]) : null;
- }
-
public TTupleDescriptor toThrift() {
TTupleDescriptor ttupleDesc = new TTupleDescriptor(id.asInt(), 0, 0);
if (table != null && table.getId() >= 0) {
@@ -242,149 +130,6 @@ public class TupleDescriptor {
return ttupleDesc;
}
- /**
- * This function is mainly used to calculate the statistics of the tuple and the layout information.
- * Generally, it occurs after the plan node materializes the slot and before calculating the plan node statistics.
- * PlanNode.init() {
- * materializedSlot();
- * tupleDesc.computeStatAndMemLayout();
- * computeStat();
- * }
- */
- public void computeStatAndMemLayout() {
- computeStat();
- computeMemLayout();
- }
-
- /**
- * This function is mainly used to evaluate the statistics of the tuple,
- * such as the average size of each row.
- * This function will be used before the computeStat() of the plan node
- * and is the pre-work for evaluating the statistics of the plan node.
- *
- * This function is theoretically only called once when the plan node is init.
- * However, the current code structure is relatively confusing
- * In order to ensure that even if it is wrongly called a second time, no error will occur,
- * so it will be initialized again at the beginning of the function.
- *
- * @deprecated In the future this function will be changed to a private function.
- */
- @Deprecated
- public void computeStat() {
- // init stat
- avgSerializedSize = 0;
-
- // compute stat
- for (SlotDescriptor d : slots) {
- if (!d.isMaterialized()) {
- continue;
- }
- ColumnStats stats = d.getStats();
- if (stats.hasAvgSerializedSize()) {
- avgSerializedSize += d.getStats().getAvgSerializedSize();
- } else {
- // TODO: for computed slots, try to come up with stats estimates
- avgSerializedSize += d.getType().getSlotSize();
- }
- }
- }
-
- /**
- * @deprecated In the future this function will be changed to a private function.
- */
- @Deprecated
- public void computeMemLayout() {
- // sort slots by size
- List<List<SlotDescriptor>> slotsBySize = Lists.newArrayListWithCapacity(PrimitiveType.getMaxSlotSize());
- for (int i = 0; i <= PrimitiveType.getMaxSlotSize(); ++i) {
- slotsBySize.add(new ArrayList<SlotDescriptor>());
- }
-
- // populate slotsBySize; also compute avgSerializedSize
- for (SlotDescriptor d : slots) {
- if (d.isMaterialized()) {
- slotsBySize.get(d.getType().getSlotSize()).add(d);
- }
- }
- // we shouldn't have anything of size 0
- Preconditions.checkState(slotsBySize.get(0).isEmpty());
-
- // slotIdx is the index into the resulting tuple struct. The first (smallest) field
- // is 0, next is 1, etc.
- int slotIdx = 0;
- for (int slotSize = 1; slotSize <= PrimitiveType.getMaxSlotSize(); ++slotSize) {
- if (slotsBySize.get(slotSize).isEmpty()) {
- continue;
- }
-
- for (SlotDescriptor d : slotsBySize.get(slotSize)) {
- d.setByteSize(slotSize);
- d.setSlotIdx(slotIdx++);
- byteSize += slotSize;
- }
- }
- }
-
- /**
- * Returns true if tuples of type 'this' can be assigned to tuples of type 'desc'
- * (checks that both have the same number of slots and that slots are of the same type)
- */
- public boolean isCompatible(TupleDescriptor desc) {
- if (slots.size() != desc.slots.size()) {
- return false;
- }
- for (int i = 0; i < slots.size(); ++i) {
- if (slots.get(i).getType() != desc.slots.get(i).getType()) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Materialize all slots.
- */
- public void materializeSlots() {
- for (SlotDescriptor slot : slots) {
- slot.setIsMaterialized(true);
- }
- }
-
- public void getTableIdToColumnNames(Map<Long, Set<String>> tableIdToColumnNames) {
- for (SlotDescriptor slotDescriptor : slots) {
- if (!slotDescriptor.isMaterialized()) {
- continue;
- }
- if (slotDescriptor.getColumn() != null) {
- TupleDescriptor parent = slotDescriptor.getParent();
- Preconditions.checkState(parent != null);
- TableIf table = parent.getTable();
- Preconditions.checkState(table != null);
- Long tableId = table.getId();
- Set<String> columnNames = tableIdToColumnNames.get(tableId);
- if (columnNames == null) {
- columnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
- tableIdToColumnNames.put(tableId, columnNames);
- }
- columnNames.add(slotDescriptor.getColumn().getName());
- } else {
- for (Expr expr : slotDescriptor.getSourceExprs()) {
- expr.getTableIdToColumnNames(tableIdToColumnNames);
- }
- }
- }
- }
-
- public Set<String> getColumnNames() {
- Map<Long, Set<String>> columnNamesInQueryOutput = Maps.newHashMap();
- getTableIdToColumnNames(columnNamesInQueryOutput);
- Set<String> columnNames = Sets.newHashSet();
- for (Set<String> names : columnNamesInQueryOutput.values()) {
- columnNames.addAll(names);
- }
- return columnNames;
- }
-
@Override
public String toString() {
String tblStr = (table == null ? "null" : table.getName());
@@ -393,7 +138,7 @@ public class TupleDescriptor {
slotStrings.add(slot.debugString());
}
return MoreObjects.toStringHelper(this).add("id", id.asInt()).add("tbl", tblStr)
- .add("is_materialized", isMaterialized).add("slots", "[" + Joiner.on(", ").join(slotStrings) + "]")
+ .add("slots", "[" + Joiner.on(", ").join(slotStrings) + "]")
.toString();
}
@@ -409,7 +154,6 @@ public class TupleDescriptor {
.add("id", id.asInt())
.add("name", debugName)
.add("tbl", tblStr)
- .add("is_materialized", isMaterialized)
.add("slots", "[" + Joiner.on(", ").join(slotStrings) + "]")
.toString();
}
@@ -424,9 +168,7 @@ public class TupleDescriptor {
.add("tbl", tblStr));
builder.append("\n");
for (SlotDescriptor slot : slots) {
- if (slot.isMaterialized()) {
- builder.append(slot.getExplainString(prefix)).append("\n");
- }
+ builder.append(slot.getExplainString(prefix)).append("\n");
}
return builder.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnStats.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnStats.java
deleted file mode 100644
index aacf1a2ebdf..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnStats.java
+++ /dev/null
@@ -1,134 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.catalog;
-
-import com.google.common.base.MoreObjects;
-import com.google.gson.annotations.SerializedName;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.Objects;
-
-/**
- * Statistics for a single column.
- */
-public class ColumnStats {
- private static final Logger LOG = LogManager.getLogger(ColumnStats.class);
-
- @SerializedName(value = "avgSerializedSize")
- private float avgSerializedSize; // in bytes; includes serialization overhead
- @SerializedName(value = "maxSize")
- private long maxSize; // in bytes
- @SerializedName(value = "numDistinctValues")
- private long numDistinctValues;
- @SerializedName(value = "numNulls")
- private long numNulls;
-
- /**
- * For fixed-length type (those which don't need additional storage besides
- * the slot they occupy), sets avgSerializedSize and maxSize to their slot size.
- */
- public ColumnStats() {
- avgSerializedSize = -1;
- maxSize = -1;
- numDistinctValues = -1;
- numNulls = -1;
- }
-
- public ColumnStats(ColumnStats other) {
- avgSerializedSize = other.avgSerializedSize;
- maxSize = other.maxSize;
- numDistinctValues = other.numDistinctValues;
- numNulls = other.numNulls;
- }
-
- public long getNumDistinctValues() {
- return numDistinctValues;
- }
-
- public void setNumDistinctValues(long numDistinctValues) {
- this.numDistinctValues = numDistinctValues;
- }
-
- public float getAvgSerializedSize() {
- return avgSerializedSize;
- }
-
- public void setAvgSerializedSize(float avgSize) {
- this.avgSerializedSize = avgSize;
- }
-
- public long getMaxSize() {
- return maxSize;
- }
-
- public void setMaxSize(long maxSize) {
- this.maxSize = maxSize;
- }
-
- public boolean hasNulls() {
- return numNulls > 0;
- }
-
- public long getNumNulls() {
- return numNulls;
- }
-
- public void setNumNulls(long numNulls) {
- this.numNulls = numNulls;
- }
-
- public boolean hasAvgSerializedSize() {
- return avgSerializedSize >= 0;
- }
-
- public boolean hasMaxSize() {
- return maxSize >= 0;
- }
-
- public boolean hasNumDistinctValues() {
- return numDistinctValues >= 0;
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this.getClass()).add("avgSerializedSize",
- avgSerializedSize).add("maxSize", maxSize).add("numDistinct", numDistinctValues).add(
- "numNulls", numNulls).toString();
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(avgSerializedSize, maxSize, numDistinctValues, numNulls);
- }
-
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (!(obj instanceof ColumnStats)) {
- return false;
- }
-
- ColumnStats stats = (ColumnStats) obj;
- return (numDistinctValues == stats.numDistinctValues)
- && (avgSerializedSize == stats.avgSerializedSize)
- && (maxSize == stats.maxSize)
- && (numNulls == stats.numNulls);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index e51891bbe9e..632b1d9e583 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -148,9 +148,6 @@ public abstract class FileQueryScanNode extends FileScanNode {
List<Column> columns = desc.getTable().getBaseSchema(false);
params.setNumOfColumnsFromFile(columns.size() - partitionKeys.size());
for (SlotDescriptor slot : desc.getSlots()) {
- if (!slot.isMaterialized()) {
- continue;
- }
TFileScanSlotInfo slotInfo = new TFileScanSlotInfo();
slotInfo.setSlotId(slot.getId().asInt());
slotInfo.setIsFileSlot(!partitionKeys.contains(slot.getColumn().getName()));
@@ -165,10 +162,6 @@ public abstract class FileQueryScanNode extends FileScanNode {
private void updateRequiredSlots() throws UserException {
params.unsetRequiredSlots();
for (SlotDescriptor slot : desc.getSlots()) {
- if (!slot.isMaterialized()) {
- continue;
- }
-
TFileScanSlotInfo slotInfo = new TFileScanSlotInfo();
slotInfo.setSlotId(slot.getId().asInt());
slotInfo.setIsFileSlot(!getPathPartitionKeys().contains(slot.getColumn().getName()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/source/RemoteDorisScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/source/RemoteDorisScanNode.java
index e6b952c2e30..41476afd03f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/source/RemoteDorisScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/source/RemoteDorisScanNode.java
@@ -201,9 +201,6 @@ public class RemoteDorisScanNode extends FileQueryScanNode {
private void createColumns() {
columns.clear();
for (SlotDescriptor slot : desc.getSlots()) {
- if (!slot.isMaterialized()) {
- continue;
- }
Column col = slot.getColumn();
columns.add("`" + col.getName() + "`");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java
index bfb1018eb40..d809045373e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java
@@ -152,9 +152,6 @@ public class JdbcScanNode extends ExternalScanNode {
private void createJdbcColumns() {
columns.clear();
for (SlotDescriptor slot : desc.getSlots()) {
- if (!slot.isMaterialized()) {
- continue;
- }
Column col = slot.getColumn();
columns.add(tbl.getProperRemoteColumnName(jdbcType, col.getName()));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java
index 64aaaaf3c1e..aa8d105959d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java
@@ -158,9 +158,6 @@ public class OdbcScanNode extends ExternalScanNode {
private void createOdbcColumns() {
columns.clear();
for (SlotDescriptor slot : desc.getSlots()) {
- if (!slot.isMaterialized()) {
- continue;
- }
Column col = slot.getColumn();
columns.add(JdbcTable.databaseProperName(odbcType, col.getName()));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
index 4cfbc0139cc..922be806d6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
@@ -252,9 +252,6 @@ public class TrinoConnectorScanNode extends FileQueryScanNode {
List<ColumnHandle> columnHandles = new ArrayList<>();
List<ColumnMetadata> columnMetadataList = new ArrayList<>();
for (SlotDescriptor slotDescriptor : source.getDesc().getSlots()) {
- if (!slotDescriptor.isMaterialized()) {
- continue;
- }
String colName = slotDescriptor.getColumn().getName();
if (columnMetadataMap.containsKey(colName)) {
columnMetadataList.add(columnMetadataMap.get(colName));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/MetadataScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/MetadataScanNode.java
index aa37bb34dc1..ef84782d1db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/MetadataScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/MetadataScanNode.java
@@ -69,7 +69,6 @@ public class MetadataScanNode extends ExternalScanNode {
@Override
protected void createScanRangeLocations() {
List<String> requiredFields = desc.getSlots().stream()
- .filter(slot -> slot.isMaterialized())
.map(slot -> slot.getColumn().getName())
.collect(java.util.stream.Collectors.toList());
TMetaScanRange metaScanRange = tvf.getMetaScanRange(requiredFields);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index afac9320543..ed5e98664cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -245,6 +245,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -298,8 +299,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
rootFragment.setOutputExprs(outputExprs);
}
Collections.reverse(context.getPlanFragments());
- // TODO: maybe we need to trans nullable directly? and then we could remove call computeMemLayout
- context.getDescTable().computeMemLayout();
if (context.getSessionVariable() != null && context.getSessionVariable().forbidUnknownColStats) {
Set<ScanNode> scans = context.getScanNodeWithUnknownColumnStats();
if (!scans.isEmpty()) {
@@ -546,7 +545,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
List<Column> targetTableColumns = hiveTableSink.getTargetTable().getFullSchema();
for (Column column : targetTableColumns) {
SlotDescriptor slotDesc = context.addSlotDesc(hiveTuple);
- slotDesc.setIsMaterialized(true);
slotDesc.setType(column.getType());
slotDesc.setColumn(column);
slotDesc.setIsNullable(column.isAllowNull());
@@ -567,7 +565,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
List<Column> targetTableColumns = icebergTableSink.getTargetTable().getFullSchema();
for (Column column : targetTableColumns) {
SlotDescriptor slotDesc = context.addSlotDesc(hiveTuple);
- slotDesc.setIsMaterialized(true);
slotDesc.setType(column.getType());
slotDesc.setColumn(column);
slotDesc.setIsNullable(column.isAllowNull());
@@ -1683,9 +1680,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
|| joinType == JoinType.LEFT_SEMI_JOIN
|| joinType == JoinType.NULL_AWARE_LEFT_ANTI_JOIN)) {
for (SlotDescriptor leftSlotDescriptor : leftSlotDescriptors) {
- if (!leftSlotDescriptor.isMaterialized()) {
- continue;
- }
SlotReference sf = leftChildOutputMap.get(context.findExprId(leftSlotDescriptor.getId()));
SlotDescriptor sd;
if (sf == null && leftSlotDescriptor.getColumn().getName().equals(Column.ROWID_COL)) {
@@ -1703,9 +1697,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
} else if (hashJoin.getOtherJoinConjuncts().isEmpty() && (isHashJoinConjunctsEmpty != isMarkJoinConjunctsEmpty)
&& (joinType == JoinType.RIGHT_ANTI_JOIN || joinType == JoinType.RIGHT_SEMI_JOIN)) {
for (SlotDescriptor rightSlotDescriptor : rightSlotDescriptors) {
- if (!rightSlotDescriptor.isMaterialized()) {
- continue;
- }
SlotReference sf = rightChildOutputMap.get(context.findExprId(rightSlotDescriptor.getId()));
SlotDescriptor sd;
if (sf == null && rightSlotDescriptor.getColumn().getName().equals(Column.ROWID_COL)) {
@@ -1722,9 +1713,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
}
} else {
for (SlotDescriptor leftSlotDescriptor : leftSlotDescriptors) {
- if (!leftSlotDescriptor.isMaterialized()) {
- continue;
- }
SlotReference sf = leftChildOutputMap.get(context.findExprId(leftSlotDescriptor.getId()));
SlotDescriptor sd;
if (sf == null && leftSlotDescriptor.getColumn().getName().equals(Column.ROWID_COL)) {
@@ -1741,9 +1729,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
leftIntermediateSlotDescriptor.add(sd);
}
for (SlotDescriptor rightSlotDescriptor : rightSlotDescriptors) {
- if (!rightSlotDescriptor.isMaterialized()) {
- continue;
- }
SlotReference sf = rightChildOutputMap.get(context.findExprId(rightSlotDescriptor.getId()));
SlotDescriptor sd;
if (sf == null && rightSlotDescriptor.getColumn().getName().equals(Column.ROWID_COL)) {
@@ -1918,9 +1903,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// TODO: because of the limitation of be, the VNestedLoopJoinNode will output column from both children
// in the intermediate tuple, so fe have to do the same, if be fix the problem, we can change it back.
for (SlotDescriptor leftSlotDescriptor : leftSlotDescriptors) {
- if (!leftSlotDescriptor.isMaterialized()) {
- continue;
- }
SlotReference sf = leftChildOutputMap.get(context.findExprId(leftSlotDescriptor.getId()));
SlotDescriptor sd;
if (sf == null && leftSlotDescriptor.getColumn().getName().equals(Column.ROWID_COL)) {
@@ -1932,9 +1914,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
leftIntermediateSlotDescriptor.add(sd);
}
for (SlotDescriptor rightSlotDescriptor : rightSlotDescriptors) {
- if (!rightSlotDescriptor.isMaterialized()) {
- continue;
- }
SlotReference sf = rightChildOutputMap.get(context.findExprId(rightSlotDescriptor.getId()));
SlotDescriptor sd;
if (sf == null && rightSlotDescriptor.getColumn().getName().equals(Column.ROWID_COL)) {
@@ -2729,9 +2708,11 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
.map(context::findSlotRef).filter(Objects::nonNull).map(SlotRef::getSlotId)
.collect(Collectors.toSet());
- for (SlotDescriptor slot : olapScanNode.getTupleDesc().getSlots()) {
+ Iterator<SlotDescriptor> it = olapScanNode.getTupleDesc().getSlots().iterator();
+ while (it.hasNext()) {
+ SlotDescriptor slot = it.next();
if (!scanIds.contains(slot.getId())) {
- slot.setIsMaterialized(false);
+ it.remove();
}
}
context.createSlotDesc(olapScanNode.getTupleDesc(), lazyScan.getRowId(), lazyScan.getTable());
@@ -3093,19 +3074,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
}
}
- private void injectRowIdColumnSlot(TupleDescriptor tupleDesc) {
- SlotDescriptor slotDesc = context.addSlotDesc(tupleDesc);
- if (LOG.isDebugEnabled()) {
- LOG.debug("inject slot {}", slotDesc);
- }
- String name = Column.ROWID_COL;
- Column col = new Column(name, Type.STRING, false, null, false, "", "rowid column");
- slotDesc.setType(Type.STRING);
- slotDesc.setColumn(col);
- slotDesc.setIsNullable(false);
- slotDesc.setIsMaterialized(true);
- }
-
/**
* topN opt: using storage data ordering to accelerate topn operation.
* refer pr: optimize topn query if order by columns is prefix of sort keys of table (#10694)
@@ -3266,12 +3234,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
List<Expr> exprList = Lists.newArrayList();
Preconditions.checkState(resultExpressionList.size() == resultExprSlots.size());
for (int j = 0; j < resultExpressionList.size(); ++j) {
- if (resultExprSlots.get(j).isMaterialized()) {
- exprList.add(ExpressionTranslator.translate(resultExpressionList.get(j), context));
- // TODO: reconsider this, we may change nullable info in previous nereids rules not here.
- resultExprSlots.get(j)
- .setIsNullable(resultExprSlots.get(j).getIsNullable() || exprList.get(j).isNullable());
- }
+ exprList.add(ExpressionTranslator.translate(resultExpressionList.get(j), context));
+ // TODO: reconsider this, we may change nullable info in previous nereids rules not here.
+ resultExprSlots.get(j)
+ .setIsNullable(resultExprSlots.get(j).getIsNullable() || exprList.get(j).isNullable());
}
materializedResultExprLists.add(exprList);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index ca0c1333613..cea2b60933f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -299,7 +299,6 @@ public class PlanTranslatorContext {
slotDescriptor.setCaptionAndNormalize(slotReference.toString());
}
slotDescriptor.setType(slotReference.getDataType().toCatalogDataType());
- slotDescriptor.setIsMaterialized(true);
SlotRef slotRef;
if (slotReference instanceof VirtualSlotReference) {
slotRef = new VirtualSlotRef(slotDescriptor);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index a6fd73ad177..8ccd7163640 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -672,12 +672,6 @@ public class OlapScanNode extends ScanNode {
bucketSeq2Bytes.merge(bucketSeq, oneReplicaBytes, Long::sum);
scanRangeLocations.add(locations);
}
-
- if (tablets.isEmpty()) {
- desc.setCardinality(0);
- } else {
- desc.setCardinality(cardinality);
- }
}
private String fastToString(long version) {
@@ -731,7 +725,6 @@ public class OlapScanNode extends ScanNode {
protected void createScanRangeLocations() throws UserException {
scanRangeLocations = Lists.newArrayList();
if (selectedPartitionIds.isEmpty()) {
- desc.setCardinality(0);
return;
}
Preconditions.checkState(selectedIndexId != -1);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ResultFileSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ResultFileSink.java
index 2f8f9ae8ab1..9059d62e1d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ResultFileSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ResultFileSink.java
@@ -156,17 +156,14 @@ public class ResultFileSink extends DataSink {
public static TupleDescriptor constructFileStatusTupleDesc(DescriptorTable descriptorTable) {
TupleDescriptor resultFileStatusTupleDesc =
descriptorTable.createTupleDescriptor("result_file_status");
- resultFileStatusTupleDesc.setIsMaterialized(true);
for (int i = 0; i < OutFileClause.RESULT_COL_NAMES.size(); ++i) {
SlotDescriptor slotDescriptor = descriptorTable.addSlotDescriptor(resultFileStatusTupleDesc);
slotDescriptor.setLabel(OutFileClause.RESULT_COL_NAMES.get(i));
slotDescriptor.setType(OutFileClause.RESULT_COL_TYPES.get(i));
slotDescriptor.setColumn(new Column(OutFileClause.RESULT_COL_NAMES.get(i),
OutFileClause.RESULT_COL_TYPES.get(i)));
- slotDescriptor.setIsMaterialized(true);
slotDescriptor.setIsNullable(false);
}
- resultFileStatusTupleDesc.computeStatAndMemLayout();
return resultFileStatusTupleDesc;
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnStatTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnStatTest.java
deleted file mode 100644
index 5e48f91069e..00000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnStatTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.catalog;
-
-import org.apache.doris.common.io.Text;
-import org.apache.doris.persist.gson.GsonUtils;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-public class ColumnStatTest {
-
- @Test
- public void testSerialization() throws Exception {
- // 1. Write objects to file
- Path path = Files.createFile(Paths.get("./columnStats"));
- DataOutputStream dos = new DataOutputStream(Files.newOutputStream(path));
-
- ColumnStats stats1 = new ColumnStats();
- Text.writeString(dos, GsonUtils.GSON.toJson(stats1));
-
- ColumnStats stats2 = new ColumnStats();
- stats2.setAvgSerializedSize(1.1f);
- stats2.setNumDistinctValues(100L);
- stats2.setMaxSize(1000L);
- stats2.setNumNulls(10000L);
- Text.writeString(dos, GsonUtils.GSON.toJson(stats2));
-
- ColumnStats stats3 = new ColumnStats();
- stats3.setAvgSerializedSize(3.3f);
- stats3.setNumDistinctValues(200L);
- stats3.setMaxSize(2000L);
- stats3.setNumNulls(20000L);
- Text.writeString(dos, GsonUtils.GSON.toJson(stats3));
-
- ColumnStats stats4 = new ColumnStats(stats3);
- Text.writeString(dos, GsonUtils.GSON.toJson(stats4));
-
- dos.flush();
- dos.close();
-
- // 2. Read objects from file
- DataInputStream dis = new DataInputStream(Files.newInputStream(path));
- ColumnStats rStats1 = GsonUtils.GSON.fromJson(Text.readString(dis), ColumnStats.class);
- Assert.assertEquals(rStats1, stats1);
-
- ColumnStats rStats2 = GsonUtils.GSON.fromJson(Text.readString(dis), ColumnStats.class);
- Assert.assertEquals(rStats2, stats2);
-
- ColumnStats rStats3 = GsonUtils.GSON.fromJson(Text.readString(dis), ColumnStats.class);
- Assert.assertEquals(rStats3, stats3);
-
- ColumnStats rStats4 = GsonUtils.GSON.fromJson(Text.readString(dis), ColumnStats.class);
- Assert.assertEquals(rStats4, stats4);
- Assert.assertEquals(rStats4, stats3);
-
- Assert.assertEquals(rStats3, rStats3);
- Assert.assertNotEquals(rStats3, this);
- Assert.assertNotEquals(rStats2, rStats3);
-
- // 3. delete files
- dis.close();
- Files.deleteIfExists(path);
- }
-
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java
index a31b1abdaa1..e3ac7aff29c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java
@@ -84,6 +84,6 @@ public class PhysicalPlanTranslatorTest {
PlanNode planNode = fragment.getPlanRoot();
List<OlapScanNode> scanNodeList = new ArrayList<>();
planNode.collect(OlapScanNode.class::isInstance, scanNodeList);
- Assertions.assertEquals(2, scanNodeList.get(0).getTupleDesc().getMaterializedSlots().size());
+ Assertions.assertEquals(2, scanNodeList.get(0).getTupleDesc().getSlots().size());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org