You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/29 04:37:34 UTC
[doris] 03/05: [bug](jdbc) fix jdbc external table with char type length error (#15386)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d2fa5078b9e125830d11b49c8f6d9653e87b05a5
Author: zhangstar333 <87...@users.noreply.github.com>
AuthorDate: Thu Dec 29 11:19:03 2022 +0800
[bug](jdbc) fix jdbc external table with char type length error (#15386)
Now have test pg and oracle with char(100), if data='abc'
but read string data length is 100, so need trim extral spaces
---
be/src/vec/exec/scan/new_jdbc_scan_node.cpp | 7 ++++---
be/src/vec/exec/scan/new_jdbc_scan_node.h | 1 +
be/src/vec/exec/scan/new_jdbc_scanner.cpp | 7 +++++--
be/src/vec/exec/scan/new_jdbc_scanner.h | 6 ++++--
be/src/vec/exec/vjdbc_connector.cpp | 13 ++++++++++++-
be/src/vec/exec/vjdbc_connector.h | 1 +
.../main/java/org/apache/doris/planner/JdbcScanNode.java | 1 +
gensrc/thrift/PlanNodes.thrift | 1 +
8 files changed, 29 insertions(+), 8 deletions(-)
diff --git a/be/src/vec/exec/scan/new_jdbc_scan_node.cpp b/be/src/vec/exec/scan/new_jdbc_scan_node.cpp
index 15ddd607ba..955a33970d 100644
--- a/be/src/vec/exec/scan/new_jdbc_scan_node.cpp
+++ b/be/src/vec/exec/scan/new_jdbc_scan_node.cpp
@@ -25,7 +25,8 @@ NewJdbcScanNode::NewJdbcScanNode(ObjectPool* pool, const TPlanNode& tnode,
: VScanNode(pool, tnode, descs),
_table_name(tnode.jdbc_scan_node.table_name),
_tuple_id(tnode.jdbc_scan_node.tuple_id),
- _query_string(tnode.jdbc_scan_node.query_string) {
+ _query_string(tnode.jdbc_scan_node.query_string),
+ _table_type(tnode.jdbc_scan_node.table_type) {
_output_tuple_id = tnode.jdbc_scan_node.tuple_id;
}
@@ -49,8 +50,8 @@ Status NewJdbcScanNode::_init_scanners(std::list<VScanner*>* scanners) {
if (_eos == true) {
return Status::OK();
}
- NewJdbcScanner* scanner =
- new NewJdbcScanner(_state, this, _limit_per_scanner, _tuple_id, _query_string);
+ NewJdbcScanner* scanner = new NewJdbcScanner(_state, this, _limit_per_scanner, _tuple_id,
+ _query_string, _table_type);
_scanner_pool.add(scanner);
RETURN_IF_ERROR(scanner->prepare(_state, _vconjunct_ctx_ptr.get()));
scanners->push_back(static_cast<VScanner*>(scanner));
diff --git a/be/src/vec/exec/scan/new_jdbc_scan_node.h b/be/src/vec/exec/scan/new_jdbc_scan_node.h
index d527f24b12..1cc34e83f0 100644
--- a/be/src/vec/exec/scan/new_jdbc_scan_node.h
+++ b/be/src/vec/exec/scan/new_jdbc_scan_node.h
@@ -37,6 +37,7 @@ private:
std::string _table_name;
TupleId _tuple_id;
std::string _query_string;
+ TOdbcTableType::type _table_type;
};
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.cpp b/be/src/vec/exec/scan/new_jdbc_scanner.cpp
index ef9004530d..cbb9588bbd 100644
--- a/be/src/vec/exec/scan/new_jdbc_scanner.cpp
+++ b/be/src/vec/exec/scan/new_jdbc_scanner.cpp
@@ -19,13 +19,15 @@
namespace doris::vectorized {
NewJdbcScanner::NewJdbcScanner(RuntimeState* state, NewJdbcScanNode* parent, int64_t limit,
- const TupleId& tuple_id, const std::string& query_string)
+ const TupleId& tuple_id, const std::string& query_string,
+ TOdbcTableType::type table_type)
: VScanner(state, static_cast<VScanNode*>(parent), limit),
_is_init(false),
_jdbc_eos(false),
_tuple_id(tuple_id),
_query_string(query_string),
- _tuple_desc(nullptr) {}
+ _tuple_desc(nullptr),
+ _table_type(table_type) {}
Status NewJdbcScanner::prepare(RuntimeState* state, VExprContext** vconjunct_ctx_ptr) {
VLOG_CRITICAL << "NewJdbcScanner::Prepare";
@@ -63,6 +65,7 @@ Status NewJdbcScanner::prepare(RuntimeState* state, VExprContext** vconjunct_ctx
_jdbc_param.passwd = jdbc_table->jdbc_passwd();
_jdbc_param.tuple_desc = _tuple_desc;
_jdbc_param.query_string = std::move(_query_string);
+ _jdbc_param.table_type = _table_type;
_jdbc_connector.reset(new (std::nothrow) JdbcConnector(_jdbc_param));
if (_jdbc_connector == nullptr) {
diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.h b/be/src/vec/exec/scan/new_jdbc_scanner.h
index f5584cd5d0..9fa17c4116 100644
--- a/be/src/vec/exec/scan/new_jdbc_scanner.h
+++ b/be/src/vec/exec/scan/new_jdbc_scanner.h
@@ -26,12 +26,12 @@ namespace vectorized {
class NewJdbcScanner : public VScanner {
public:
NewJdbcScanner(RuntimeState* state, NewJdbcScanNode* parent, int64_t limit,
- const TupleId& tuple_id, const std::string& query_string);
+ const TupleId& tuple_id, const std::string& query_string,
+ TOdbcTableType::type table_type);
Status open(RuntimeState* state) override;
Status close(RuntimeState* state) override;
-public:
Status prepare(RuntimeState* state, VExprContext** vconjunct_ctx_ptr);
protected:
@@ -48,6 +48,8 @@ private:
std::string _query_string;
// Descriptor of tuples read from JDBC table.
const TupleDescriptor* _tuple_desc;
+ // the sql query database type: like mysql, PG...
+ TOdbcTableType::type _table_type;
// Scanner of JDBC.
std::unique_ptr<JdbcConnector> _jdbc_connector;
JdbcConnectorParam _jdbc_param;
diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp
index ce171ad569..f2302f3baa 100644
--- a/be/src/vec/exec/vjdbc_connector.cpp
+++ b/be/src/vec/exec/vjdbc_connector.cpp
@@ -418,8 +418,19 @@ Status JdbcConnector::_convert_column_data(JNIEnv* env, jobject jobj,
M(TYPE_FLOAT, float, vectorized::ColumnVector<vectorized::Float32>)
M(TYPE_DOUBLE, double, vectorized::ColumnVector<vectorized::Float64>)
#undef M
+ case TYPE_CHAR: {
+ std::string data = _jobject_to_string(env, jobj);
+ // Now have test pg and oracle with char(100), if data='abc'
+ // but read string data length is 100, so need trim extra spaces
+ if ((_conn_param.table_type == TOdbcTableType::POSTGRESQL) ||
+ (_conn_param.table_type == TOdbcTableType::ORACLE)) {
+ data = data.erase(data.find_last_not_of(' ') + 1);
+ }
+ reinterpret_cast<vectorized::ColumnString*>(col_ptr)->insert_data(data.c_str(),
+ data.length());
+ break;
+ }
case TYPE_STRING:
- case TYPE_CHAR:
case TYPE_VARCHAR: {
std::string data = _jobject_to_string(env, jobj);
reinterpret_cast<vectorized::ColumnString*>(col_ptr)->insert_data(data.c_str(),
diff --git a/be/src/vec/exec/vjdbc_connector.h b/be/src/vec/exec/vjdbc_connector.h
index e6da1a0151..84ca17e02c 100644
--- a/be/src/vec/exec/vjdbc_connector.h
+++ b/be/src/vec/exec/vjdbc_connector.h
@@ -33,6 +33,7 @@ struct JdbcConnectorParam {
std::string user;
std::string passwd;
std::string query_string;
+ TOdbcTableType::type table_type;
const TupleDescriptor* tuple_desc;
};
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java
index b9da4b45c0..c15a12371c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java
@@ -195,6 +195,7 @@ public class JdbcScanNode extends ScanNode {
msg.jdbc_scan_node.setTupleId(desc.getId().asInt());
msg.jdbc_scan_node.setTableName(tableName);
msg.jdbc_scan_node.setQueryString(getJdbcQueryStr());
+ msg.jdbc_scan_node.setTableType(jdbcType);
}
@Override
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index a7a21a0499..613c884a06 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -406,6 +406,7 @@ struct TJdbcScanNode {
1: optional Types.TTupleId tuple_id
2: optional string table_name
3: optional string query_string
+ 4: optional Types.TOdbcTableType table_type
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org