You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/05/12 05:27:09 UTC
[incubator-doris] branch master updated: [fix](broker-load) can't load parquet file with column name case sensitive with Doris column (#9358)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new cfbf13710b [fix](broker-load) can't load parquet file with column name case sensitive with Doris column (#9358)
cfbf13710b is described below
commit cfbf13710bbd895a9e66a813c88d7f019f425fa8
Author: deardeng <56...@qq.com>
AuthorDate: Wed May 11 22:27:03 2022 -0700
[fix](broker-load) can't load parquet file with column name case sensitive with Doris column (#9358)
---
fe/fe-core/src/main/java/org/apache/doris/load/Load.java | 8 ++++++--
.../src/main/java/org/apache/doris/planner/BrokerScanNode.java | 4 ++--
.../main/java/org/apache/doris/planner/StreamLoadScanNode.java | 7 +++++--
3 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
index 905f950737..ea381977e2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
@@ -1049,8 +1049,12 @@ public class Load {
for (ImportColumnDesc importColumnDesc : copiedColumnExprs) {
// make column name case match with real column name
String columnName = importColumnDesc.getColumnName();
- String realColName = tbl.getColumn(columnName) == null ? columnName
- : tbl.getColumn(columnName).getName();
+ String realColName;
+ if (tbl.getColumn(columnName) == null || importColumnDesc.getExpr() == null) {
+ realColName = columnName;
+ } else {
+ realColName = tbl.getColumn(columnName).getName();
+ }
if (importColumnDesc.getExpr() != null) {
Expr expr = transformHadoopFunctionExpr(tbl, realColName, importColumnDesc.getExpr());
exprsByName.put(realColName, expr);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java
index a7b9493b88..6f004f2c9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java
@@ -247,8 +247,8 @@ public class BrokerScanNode extends LoadScanNode {
*/
private void initColumns(ParamCreateContext context) throws UserException {
context.srcTupleDescriptor = analyzer.getDescTbl().createTupleDescriptor();
- context.slotDescByName = Maps.newHashMap();
- context.exprMap = Maps.newHashMap();
+ context.slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ context.exprMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
// for load job, column exprs is got from file group
// for query, there is no column exprs, they will be got from table's schema in "Load.initColumns"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java
index 0930bad5f0..65a265d9c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java
@@ -66,8 +66,11 @@ public class StreamLoadScanNode extends LoadScanNode {
private TupleDescriptor srcTupleDesc;
private TBrokerScanRange brokerScanRange;
- private Map<String, SlotDescriptor> slotDescByName = Maps.newHashMap();
- private Map<String, Expr> exprsByName = Maps.newHashMap();
+ // If use case sensitive map, for example,
+ // the column name 「A」 in the table and the mapping '(a) set (A = a)' in load sql,
+ // Slotdescbyname stores「a」, later will use 「a」to get table's 「A」 column info, will throw exception.
+ private final Map<String, SlotDescriptor> slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ private final Map<String, Expr> exprsByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
// used to construct for streaming loading
public StreamLoadScanNode(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org