You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/10/21 02:12:20 UTC
[incubator-doris] branch master updated: [Bug] Do not push down
limit operation when ODBC table do not push all conjunct as filter. (#4764)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 349cc9e [Bug] Do not push down limit operation when ODBC table do not push all conjunct as filter. (#4764)
349cc9e is described below
commit 349cc9ef171c2f2196d7c93c46a48fe3f35692a7
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Wed Oct 21 10:12:12 2020 +0800
[Bug] Do not push down limit operation when ODBC table do not push all conjunct as filter. (#4764)
---
be/src/exec/odbc_scan_node.cpp | 8 +++++++-
.../main/java/org/apache/doris/planner/OdbcScanNode.java | 16 +++++++++++-----
.../java/org/apache/doris/planner/QueryPlanTest.java | 9 ++++++++-
3 files changed, 26 insertions(+), 7 deletions(-)
diff --git a/be/src/exec/odbc_scan_node.cpp b/be/src/exec/odbc_scan_node.cpp
index 14ea0eb..ae30d63 100644
--- a/be/src/exec/odbc_scan_node.cpp
+++ b/be/src/exec/odbc_scan_node.cpp
@@ -141,6 +141,11 @@ Status OdbcScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eo
SCOPED_TIMER(_runtime_profile->total_time_counter());
SCOPED_TIMER(materialize_tuple_timer());
+ if (reached_limit()) {
+ *eos = true;
+ return Status::OK();
+ }
+
// create new tuple buffer for row_batch
int tuple_buffer_size = row_batch->capacity() * _tuple_desc->byte_size();
void* tuple_buffer = _tuple_pool->allocate(tuple_buffer_size);
@@ -156,10 +161,11 @@ Status OdbcScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eo
while (true) {
RETURN_IF_CANCELLED(state);
- if (row_batch->is_full()) {
+ if (reached_limit() || row_batch->is_full()) {
// hang on to last allocated chunk in pool, we'll keep writing into it in the
// next get_next() call
row_batch->tuple_data_pool()->acquire_data(_tuple_pool.get(), !reached_limit());
+ *eos = reached_limit();
return Status::OK();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
index 6a37415..0d6ce80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
@@ -64,7 +64,7 @@ public class OdbcScanNode extends ScanNode {
// Now some database have different function call like doris, now doris do not
// push down the function call except MYSQL
- private static boolean needPushDown(TOdbcTableType tableType, Expr expr) {
+ private static boolean shouldPushDownConjunct(TOdbcTableType tableType, Expr expr) {
if (!tableType.equals(TOdbcTableType.MYSQL)) {
List<FunctionCallExpr> fnExprList = Lists.newArrayList();
expr.collect(FunctionCallExpr.class, fnExprList);
@@ -114,16 +114,22 @@ public class OdbcScanNode extends ScanNode {
return output.toString();
}
+ // only all conjuncts be pushed down as filter, we can
+ // push down limit operation to ODBC table
+ private boolean shouldPushDownLimit() {
+ return limit != -1 && conjuncts.isEmpty();
+ }
+
private String getOdbcQueryStr() {
StringBuilder sql = new StringBuilder("SELECT ");
// Oracle use the where clause to do top n
- if (limit != -1 && odbcType == TOdbcTableType.ORACLE) {
+ if (shouldPushDownLimit() && odbcType == TOdbcTableType.ORACLE) {
filters.add("ROWNUM <= " + limit);
}
// MSSQL use select top to do top n
- if (limit != -1 && odbcType == TOdbcTableType.SQLSERVER) {
+ if (shouldPushDownLimit() && odbcType == TOdbcTableType.SQLSERVER) {
sql.append("TOP " + limit + " ");
}
@@ -137,7 +143,7 @@ public class OdbcScanNode extends ScanNode {
}
// Other DataBase use limit do top n
- if (limit != -1 && (odbcType == TOdbcTableType.MYSQL || odbcType == TOdbcTableType.POSTGRESQL || odbcType == TOdbcTableType.MONGODB) ) {
+ if (shouldPushDownLimit() && (odbcType == TOdbcTableType.MYSQL || odbcType == TOdbcTableType.POSTGRESQL || odbcType == TOdbcTableType.MONGODB) ) {
sql.append(" LIMIT " + limit);
}
@@ -175,7 +181,7 @@ public class OdbcScanNode extends ScanNode {
}
ArrayList<Expr> odbcConjuncts = Expr.cloneList(conjuncts, sMap);
for (Expr p : odbcConjuncts) {
- if (needPushDown(odbcType, p)) {
+ if (shouldPushDownConjunct(odbcType, p)) {
String filter = p.toMySql();
filters.add(filter);
conjuncts.remove(p);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
index ba3e24c..71a8cbb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -1223,9 +1223,16 @@ public class QueryPlanTest {
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
Assert.assertTrue(explainString.contains("LIMIT 10"));
- // ODBC table (Oracle)
+ // ODBC table (Oracle) not push down limit
queryStr = "explain select * from odbc_oracle where k1 > 10 and abs(k1) > 10 limit 10";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
+ // abs is function, so Doris do not push down function except MySQL Database
+ // so should not push down limit operation
+ Assert.assertTrue(!explainString.contains("ROWNUM <= 10"));
+
+ // ODBC table (Oracle) push down limit
+ queryStr = "explain select * from odbc_oracle where k1 > 10 limit 10";
+ explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
Assert.assertTrue(explainString.contains("ROWNUM <= 10"));
// MySQL table
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org