You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/05/16 16:39:13 UTC
[incubator-doris] 03/17: [fix](broker-load) can't load parquet file with column name case sensitive with Doris column (#9358)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 75fbc9d8adbe18232c550cce21c0fef11f92fc6c
Author: deardeng <56...@qq.com>
AuthorDate: Wed May 11 22:27:03 2022 -0700
[fix](broker-load) can't load parquet file with column name case sensitive with Doris column (#9358)
---
fe/fe-core/src/main/java/org/apache/doris/load/Load.java | 8 ++++++--
.../src/main/java/org/apache/doris/planner/BrokerScanNode.java | 4 ++--
.../main/java/org/apache/doris/planner/StreamLoadScanNode.java | 7 +++++--
3 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
index 905f950737..ea381977e2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
@@ -1049,8 +1049,12 @@ public class Load {
for (ImportColumnDesc importColumnDesc : copiedColumnExprs) {
// make column name case match with real column name
String columnName = importColumnDesc.getColumnName();
- String realColName = tbl.getColumn(columnName) == null ? columnName
- : tbl.getColumn(columnName).getName();
+ String realColName;
+ if (tbl.getColumn(columnName) == null || importColumnDesc.getExpr() == null) {
+ realColName = columnName;
+ } else {
+ realColName = tbl.getColumn(columnName).getName();
+ }
if (importColumnDesc.getExpr() != null) {
Expr expr = transformHadoopFunctionExpr(tbl, realColName, importColumnDesc.getExpr());
exprsByName.put(realColName, expr);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java
index 499ef83728..a6f26c4920 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java
@@ -240,8 +240,8 @@ public class BrokerScanNode extends LoadScanNode {
*/
private void initColumns(ParamCreateContext context) throws UserException {
context.srcTupleDescriptor = analyzer.getDescTbl().createTupleDescriptor();
- context.slotDescByName = Maps.newHashMap();
- context.exprMap = Maps.newHashMap();
+ context.slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ context.exprMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
// for load job, column exprs is got from file group
// for query, there is no column exprs, they will be got from table's schema in "Load.initColumns"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java
index 200e232e2e..9f787ecf4b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java
@@ -66,8 +66,11 @@ public class StreamLoadScanNode extends LoadScanNode {
private TupleDescriptor srcTupleDesc;
private TBrokerScanRange brokerScanRange;
- private Map<String, SlotDescriptor> slotDescByName = Maps.newHashMap();
- private Map<String, Expr> exprsByName = Maps.newHashMap();
+ // If use case sensitive map, for example,
+ // the column name 「A」 in the table and the mapping '(a) set (A = a)' in load sql，
+ // Slotdescbyname stores「a」, later will use 「a」to get table's 「A」 column info, will throw exception.
+ private final Map<String, SlotDescriptor> slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ private final Map<String, Expr> exprsByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
// used to construct for streaming loading
public StreamLoadScanNode(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org