You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/09/03 03:17:32 UTC
(doris) branch branch-2.1 updated: [fix](ES Catalog)Do not push down limit to ES when predicates can not be processed by ES. (#40111) (#40265)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 41271ecba0f [fix](ES Catalog)Do not push down limit to ES when predicates can not be processed by ES. (#40111) (#40265)
41271ecba0f is described below
commit 41271ecba0fb761ae92de7eaaea503ce28634b53
Author: qiye <ji...@gmail.com>
AuthorDate: Tue Sep 3 11:17:24 2024 +0800
[fix](ES Catalog)Do not push down limit to ES when predicates can not be processed by ES. (#40111) (#40265)
bp #40111
---
be/src/pipeline/exec/es_scan_operator.cpp | 5 +-
.../data/external_table_p0/es/test_es_query.out | 104 +++++++++++++++++++++
.../external_table_p0/es/test_es_query.groovy | 12 +++
3 files changed, 119 insertions(+), 2 deletions(-)
diff --git a/be/src/pipeline/exec/es_scan_operator.cpp b/be/src/pipeline/exec/es_scan_operator.cpp
index 0e5018b85d6..024ed6c53dc 100644
--- a/be/src/pipeline/exec/es_scan_operator.cpp
+++ b/be/src/pipeline/exec/es_scan_operator.cpp
@@ -84,9 +84,10 @@ Status EsScanLocalState::_init_scanners(std::list<vectorized::VScannerSPtr>* sca
std::to_string(vectorized::RuntimeFilterConsumer::_state->batch_size());
properties[ESScanReader::KEY_HOST_PORT] = get_host_and_port(es_scan_range->es_hosts);
// push down limit to Elasticsearch
- // if predicate in _conjunct_ctxs can not be processed by Elasticsearch, we can not push down limit operator to Elasticsearch
+ // if predicate in _conjuncts can not be processed by Elasticsearch, we can not push down limit operator to Elasticsearch
if (p.limit() != -1 &&
- p.limit() <= vectorized::RuntimeFilterConsumer::_state->batch_size()) {
+ p.limit() <= vectorized::RuntimeFilterConsumer::_state->batch_size() &&
+ p.conjuncts().empty()) {
properties[ESScanReader::KEY_TERMINATE_AFTER] = std::to_string(p.limit());
}
diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p0/es/test_es_query.out
index a2289bec1b4..1e103748dc4 100644
--- a/regression-test/data/external_table_p0/es/test_es_query.out
+++ b/regression-test/data/external_table_p0/es/test_es_query.out
@@ -193,6 +193,16 @@ I'm not null or empty
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
+-- !sql_5_21 --
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+
+-- !sql_5_22 --
+2022-08-08T12:10:10 2022-08-08 12
+2022-08-08T12:10:10 2022-08-08 12
+
-- !sql_6_02 --
[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10
@@ -265,6 +275,16 @@ I'm not null or empty
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
+-- !sql_6_21 --
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+
+-- !sql_6_22 --
+2022-08-08T12:10:10 2022-08-08 12
+2022-08-08T12:10:10 2022-08-08 12
+
-- !sql_7_02 --
[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 [...]
@@ -370,6 +390,22 @@ I'm not null or empty
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
+-- !sql_7_26 --
+2022-08-08T04:10:10 2022-08-08
+2022-08-08T04:10:10 2022-08-08
+
+-- !sql_7_27 --
+2022-08-09T12:10:10 2022-08-09
+2022-08-09T12:10:10 2022-08-09
+
+-- !sql_7_28 --
+2022-08-10T04:10:10 2022-08-10
+2022-08-10T04:10:10 2022-08-10
+
+-- !sql_7_29 --
+2022-08-11T12:10:10 2022-08-11
+2022-08-11T12:10:10 2022-08-11
+
-- !sql_7_26 --
value1 value2
@@ -475,6 +511,22 @@ I'm not null or empty
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
+-- !sql_8_24 --
+2022-08-08T04:10:10 2022-08-08
+2022-08-08T04:10:10 2022-08-08
+
+-- !sql_8_25 --
+2022-08-09T12:10:10 2022-08-09
+2022-08-09T12:10:10 2022-08-09
+
+-- !sql_8_26 --
+2022-08-10T04:10:10 2022-08-10
+2022-08-10T04:10:10 2022-08-10
+
+-- !sql_8_27 --
+2022-08-11T12:10:10 2022-08-11
+2022-08-11T12:10:10 2022-08-11
+
-- !sql01 --
["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [-1, 0, 1, 2] [0, 1, 2, 3] ["d", "e", "f"] [128, 129, -129, -130] ["192.168.0.1", "127.0.0.1"] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 ["2020-01-01", "2020-01-02"] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ["a", "b", "c"] [{"name":"Andy","age":18},{"name":"Tim","age":28}] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] \N [{"last":"Smith","first [...]
@@ -608,6 +660,16 @@ I'm not null or empty
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
+-- !sql_5_21 --
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+
+-- !sql_5_22 --
+2022-08-08T12:10:10 2022-08-08 12
+2022-08-08T12:10:10 2022-08-08 12
+
-- !sql_6_02 --
[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] \N string1 text#1 3.14 2022-08-08T00:00 12345 2022-08-08T20:10:10
@@ -680,6 +742,16 @@ I'm not null or empty
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
+-- !sql_6_21 --
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+2022-08-08T20:10:10 2022-08-08 20
+
+-- !sql_6_22 --
+2022-08-08T12:10:10 2022-08-08 12
+2022-08-08T12:10:10 2022-08-08 12
+
-- !sql_7_02 --
[1, 0, 1, 1] [1, -2, -3, 4] ["2020-01-01", "2020-01-02"] ["2020-01-01 12:00:00", "2020-01-02 13:01:01"] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] [{"name":"Andy","age":18},{"name":"Tim","age":28}] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] debug \N This string can be quite lengthy string1 2022-08-08T20:10:10 [...]
@@ -785,6 +857,22 @@ I'm not null or empty
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
+-- !sql_7_26 --
+2022-08-08T04:10:10 2022-08-08
+2022-08-08T04:10:10 2022-08-08
+
+-- !sql_7_27 --
+2022-08-09T12:10:10 2022-08-09
+2022-08-09T12:10:10 2022-08-09
+
+-- !sql_7_28 --
+2022-08-10T04:10:10 2022-08-10
+2022-08-10T04:10:10 2022-08-10
+
+-- !sql_7_29 --
+2022-08-11T12:10:10 2022-08-11
+2022-08-11T12:10:10 2022-08-11
+
-- !sql_7_26 --
value1 value2
@@ -890,3 +978,19 @@ I'm not null or empty
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
[{"name":"Andy","age":18},{"name":"Tim","age":28}] [{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}] "Andy" "White"
+-- !sql_8_24 --
+2022-08-08T04:10:10 2022-08-08
+2022-08-08T04:10:10 2022-08-08
+
+-- !sql_8_25 --
+2022-08-09T12:10:10 2022-08-09
+2022-08-09T12:10:10 2022-08-09
+
+-- !sql_8_26 --
+2022-08-10T04:10:10 2022-08-10
+2022-08-10T04:10:10 2022-08-10
+
+-- !sql_8_27 --
+2022-08-11T12:10:10 2022-08-11
+2022-08-11T12:10:10 2022-08-11
+
diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p0/es/test_es_query.groovy
index 54b1c331de5..9d3d2cb80ad 100644
--- a/regression-test/suites/external_table_p0/es/test_es_query.groovy
+++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy
@@ -214,6 +214,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql_5_18 """select message from test1 where not_null_or_empty(message)"""
order_qt_sql_5_19 """select * from test1 where esquery(c_unsigned_long, '{"match":{"c_unsigned_long":0}}')"""
order_qt_sql_5_20 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;"""
+ order_qt_sql_5_21 """select test6, substring(test6, 1, 13) from test2 where substring(test6, 1, 13) = '2022-08-08 20' limit 4;"""
+ order_qt_sql_5_22 """select test6, substring(test6, 1, 13) from test2 where substring(test6, 1, 13) = '2022-08-08 12' limit 4;"""
try {
sql """select * from composite_type_array;"""
fail("Should not reach here")
@@ -243,6 +245,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql_6_18 """select message from test1 where not_null_or_empty(message)"""
order_qt_sql_6_19 """select * from test1 where esquery(c_person, '{"match":{"c_person.name":"Andy"}}')"""
order_qt_sql_6_20 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;"""
+ order_qt_sql_6_21 """select test6, substring(test6, 1, 13) from test2 where substring(test6, 1, 13) = '2022-08-08 20' limit 4;"""
+ order_qt_sql_6_22 """select test6, substring(test6, 1, 13) from test2 where substring(test6, 1, 13) = '2022-08-08 12' limit 4;"""
try {
sql """select * from composite_type_array;"""
fail("Should not reach here")
@@ -295,6 +299,10 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql_7_23 """select * from test1 where level = 'debug'"""
order_qt_sql_7_24 """select * from test1 where esquery(c_float, '{"match":{"c_float":1.1}}')"""
order_qt_sql_7_25 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;"""
+ order_qt_sql_7_26 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-08' limit 2;"""
+ order_qt_sql_7_27 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-09' limit 2;"""
+ order_qt_sql_7_28 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-10' limit 2;"""
+ order_qt_sql_7_29 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-11' limit 2;"""
try {
sql """select * from composite_type_array;"""
fail("Should not reach here")
@@ -347,6 +355,10 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
order_qt_sql_8_21 """select * from test1 where level = 'debug'"""
order_qt_sql_8_22 """select * from test1 where esquery(c_ip, '{"match":{"c_ip":"192.168.0.1"}}')"""
order_qt_sql_8_23 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test1;"""
+ order_qt_sql_8_24 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-08' limit 2;"""
+ order_qt_sql_8_25 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-09' limit 2;"""
+ order_qt_sql_8_26 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-10' limit 2;"""
+ order_qt_sql_8_27 """select test7,substring(test7, 1, 10) from test2 where substring(test7, 1, 10)='2022-08-11' limit 2;"""
try {
sql """select * from composite_type_array;"""
fail("Should not reach here")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org