You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/25 08:57:32 UTC
[doris] branch branch-2.0 updated (3c81bca5684 -> 0c963fa548f)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 3c81bca5684 [opt](filecache) do not sync segment data into storage system #25691 (#25856)
new 2eb30166c0a [Fix](Nereids) fix column statistic derive in outer join estimation (#25586)
new 61ff95e3456 [improvement](routine-load) add routine load rows check (#25818)
new 0c963fa548f [fix](field) fix coredump of field function when the first argument is const (#25859)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../runtime/routine_load/data_consumer_group.cpp | 3 +-
be/src/runtime/routine_load/data_consumer_group.h | 6 +++
.../routine_load/routine_load_task_executor.cpp | 9 ++++
be/src/vec/functions/function_multi_same_args.h | 2 +
.../apache/doris/nereids/stats/JoinEstimation.java | 11 +++--
.../doris/nereids/stats/JoinEstimateTest.java | 54 ++++++++++++++++++++++
.../test_string_function_regexp.out | 6 +++
.../test_string_function_regexp.groovy | 3 ++
8 files changed, 90 insertions(+), 4 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/03: [fix](field) fix coredump of field function when the first argument is const (#25859)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0c963fa548f25de34a42d19e26909e70719fe866
Author: TengJianPing <18...@users.noreply.github.com>
AuthorDate: Wed Oct 25 14:14:32 2023 +0800
[fix](field) fix coredump of field function when the first argument is const (#25859)
---
be/src/vec/functions/function_multi_same_args.h | 2 ++
.../sql_functions/string_functions/test_string_function_regexp.out | 6 ++++++
.../string_functions/test_string_function_regexp.groovy | 3 +++
3 files changed, 11 insertions(+)
diff --git a/be/src/vec/functions/function_multi_same_args.h b/be/src/vec/functions/function_multi_same_args.h
index 3da03beffd3..2f44e4f2aba 100644
--- a/be/src/vec/functions/function_multi_same_args.h
+++ b/be/src/vec/functions/function_multi_same_args.h
@@ -37,6 +37,8 @@ public:
bool use_default_implementation_for_nulls() const override { return true; }
+ bool use_default_implementation_for_constants() const override { return false; }
+
bool is_variadic() const override { return true; }
size_t get_number_of_arguments() const override { return 0; }
diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
index 133076ec13b..cfe2fd3eaf7 100644
--- a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
+++ b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_regexp.out
@@ -230,3 +230,9 @@ Suzi
Suzi
Ben
+-- !sql_field4 --
+3
+
+-- !sql_field5 --
+2
+
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
index 59414f5c1b7..d25c995ea48 100644
--- a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
@@ -151,5 +151,8 @@ suite("test_string_function_regexp") {
qt_sql_field1 "select name from ${tbName2} order by field(name,'Suzi','Ben','Henry');"
qt_sql_field2 "select name from ${tbName2} order by field(name,'Ben','Henry');"
qt_sql_field3 "select name from ${tbName2} order by field(name,'Henry') desc,id;"
+
+ qt_sql_field4 "SELECT FIELD('21','2130', '2131', '21');"
+ qt_sql_field5 "SELECT FIELD(21, 2130, 21, 2131);"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/03: [improvement](routine-load) add routine load rows check (#25818)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 61ff95e34564e28d2963f17402ce1150c18322e3
Author: HHoflittlefish777 <77...@users.noreply.github.com>
AuthorDate: Wed Oct 25 11:04:28 2023 +0800
[improvement](routine-load) add routine load rows check (#25818)
---
be/src/runtime/routine_load/data_consumer_group.cpp | 3 ++-
be/src/runtime/routine_load/data_consumer_group.h | 6 ++++++
be/src/runtime/routine_load/routine_load_task_executor.cpp | 9 +++++++++
3 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/be/src/runtime/routine_load/data_consumer_group.cpp b/be/src/runtime/routine_load/data_consumer_group.cpp
index 27eea942664..9274ccdb419 100644
--- a/be/src/runtime/routine_load/data_consumer_group.cpp
+++ b/be/src/runtime/routine_load/data_consumer_group.cpp
@@ -125,9 +125,10 @@ Status KafkaDataConsumerGroup::start_all(std::shared_ptr<StreamLoadContext> ctx,
bool eos = false;
while (true) {
if (eos || left_time <= 0 || left_rows <= 0 || left_bytes <= 0) {
+ _rows = ctx->max_batch_rows - left_rows;
LOG(INFO) << "consumer group done: " << _grp_id
<< ". consume time(ms)=" << ctx->max_interval_s * 1000 - left_time
- << ", received rows=" << ctx->max_batch_rows - left_rows
+ << ", received rows=" << _rows
<< ", received bytes=" << ctx->max_batch_size - left_bytes << ", eos: " << eos
<< ", left_time: " << left_time << ", left_rows: " << left_rows
<< ", left_bytes: " << left_bytes
diff --git a/be/src/runtime/routine_load/data_consumer_group.h b/be/src/runtime/routine_load/data_consumer_group.h
index e15ad7115f6..0cda80a9ec4 100644
--- a/be/src/runtime/routine_load/data_consumer_group.h
+++ b/be/src/runtime/routine_load/data_consumer_group.h
@@ -60,6 +60,10 @@ public:
++_counter;
}
+ int64_t get_consumer_rows() const { return _rows; }
+
+ void set_consumer_rows(int64_t rows) { _rows = rows; }
+
// start all consumers
virtual Status start_all(std::shared_ptr<StreamLoadContext> ctx,
std::shared_ptr<io::KafkaConsumerPipe> kafka_pipe) {
@@ -77,6 +81,8 @@ protected:
// when the counter becomes zero, shutdown the queue to finish
std::mutex _mutex;
int _counter;
+ // received total rows
+ int64_t _rows {0};
};
// for kafka
diff --git a/be/src/runtime/routine_load/routine_load_task_executor.cpp b/be/src/runtime/routine_load/routine_load_task_executor.cpp
index 57a847515b9..d8a0dc6a289 100644
--- a/be/src/runtime/routine_load/routine_load_task_executor.cpp
+++ b/be/src/runtime/routine_load/routine_load_task_executor.cpp
@@ -352,6 +352,15 @@ void RoutineLoadTaskExecutor::exec_task(std::shared_ptr<StreamLoadContext> ctx,
// wait for all consumers finished
HANDLE_ERROR(ctx->future.get(), "consume failed");
+ // check received and load rows
+ LOG(INFO) << "routine load task received rows: " << consumer_grp.get()->get_consumer_rows()
+ << " load total rows: " << ctx.get()->number_total_rows
+ << " loaded rows: " << ctx.get()->number_loaded_rows
+ << " filtered rows: " << ctx.get()->number_filtered_rows
+ << " unselected rows: " << ctx.get()->number_unselected_rows;
+ DCHECK(consumer_grp.get()->get_consumer_rows() == ctx.get()->number_total_rows);
+ consumer_grp.get()->set_consumer_rows(0);
+
ctx->load_cost_millis = UnixMillis() - ctx->start_millis;
// return the consumer back to pool
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/03: [Fix](Nereids) fix column statistic derive in outer join estimation (#25586)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2eb30166c0ab6dbcbacf18f956e3871c522009bd
Author: LiBinfeng <46...@users.noreply.github.com>
AuthorDate: Tue Oct 24 22:25:20 2023 +0800
[Fix](Nereids) fix column statistic derive in outer join estimation (#25586)
Problem:
When join estimation, upper join output slot statistic ndv would go wrong
Example:
we have two table:
tableA (a1[ndv = 10.0]) tableB(b1[ndv = 0.0], b2[ndv = 10.0])
tableA left join tableB on A.a1 = B.b1. which B.b1 with ndv zero.
the problem is after join estimation, B.b2 changed to 1.0.
Reason:
When estimating outer join, we can assume it behave like inner join. But we estimation then like inner join do
Solved:
When estimation outer join, output slot would update seperatly.
---
.../apache/doris/nereids/stats/JoinEstimation.java | 11 +++--
.../doris/nereids/stats/JoinEstimateTest.java | 54 ++++++++++++++++++++++
2 files changed, 62 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
index ef4575e3308..800886c177f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
@@ -279,6 +279,11 @@ public class JoinEstimation {
*/
public static Statistics estimate(Statistics leftStats, Statistics rightStats, Join join) {
JoinType joinType = join.getJoinType();
+ Statistics crossJoinStats = new StatisticsBuilder()
+ .setRowCount(Math.max(1, leftStats.getRowCount()) * Math.max(1, rightStats.getRowCount()))
+ .putColumnStatistics(leftStats.columnStatistics())
+ .putColumnStatistics(rightStats.columnStatistics())
+ .build();
if (joinType.isSemiOrAntiJoin()) {
return estimateSemiOrAnti(leftStats, rightStats, join);
} else if (joinType == JoinType.INNER_JOIN) {
@@ -289,15 +294,15 @@ public class JoinEstimation {
Statistics innerJoinStats = estimateInnerJoin(leftStats, rightStats, join);
double rowCount = Math.max(leftStats.getRowCount(), innerJoinStats.getRowCount());
rowCount = Math.max(leftStats.getRowCount(), rowCount);
- return innerJoinStats.withRowCountAndEnforceValid(rowCount);
+ return crossJoinStats.withRowCountAndEnforceValid(rowCount);
} else if (joinType == JoinType.RIGHT_OUTER_JOIN) {
Statistics innerJoinStats = estimateInnerJoin(leftStats, rightStats, join);
double rowCount = Math.max(rightStats.getRowCount(), innerJoinStats.getRowCount());
rowCount = Math.max(rowCount, rightStats.getRowCount());
- return innerJoinStats.withRowCountAndEnforceValid(rowCount);
+ return crossJoinStats.withRowCountAndEnforceValid(rowCount);
} else if (joinType == JoinType.FULL_OUTER_JOIN) {
Statistics innerJoinStats = estimateInnerJoin(leftStats, rightStats, join);
- return innerJoinStats.withRowCountAndEnforceValid(leftStats.getRowCount()
+ return crossJoinStats.withRowCountAndEnforceValid(leftStats.getRowCount()
+ rightStats.getRowCount() + innerJoinStats.getRowCount());
} else if (joinType == JoinType.CROSS_JOIN) {
return new StatisticsBuilder()
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java
index f49bf94e2d3..2735e26da46 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java
@@ -91,4 +91,58 @@ public class JoinEstimateTest {
Assertions.assertNotNull(outAStats);
Assertions.assertEquals(5, outBStats.ndv);
}
+
+ @Test
+ public void testOuterJoinStats() {
+ SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+ SlotReference b = new SlotReference("b", IntegerType.INSTANCE);
+ SlotReference c = new SlotReference("c", IntegerType.INSTANCE);
+ EqualTo eq = new EqualTo(a, b);
+ Statistics leftStats = new StatisticsBuilder().setRowCount(100).build();
+ leftStats.addColumnStats(a,
+ new ColumnStatisticBuilder()
+ .setCount(100)
+ .setNdv(10)
+ .build()
+ );
+ Statistics rightStats = new StatisticsBuilder().setRowCount(80).build();
+ rightStats.addColumnStats(b,
+ new ColumnStatisticBuilder()
+ .setCount(80)
+ .setNdv(0)
+ .build()
+ ).addColumnStats(c,
+ new ColumnStatisticBuilder()
+ .setCount(80)
+ .setNdv(20)
+ .build()
+ );
+ IdGenerator<GroupId> idGenerator = GroupId.createGenerator();
+ GroupPlan left = new GroupPlan(new Group(idGenerator.getNextId(), new LogicalProperties(
+ new Supplier<List<Slot>>() {
+ @Override
+ public List<Slot> get() {
+ return Lists.newArrayList(a);
+ }
+ })));
+ GroupPlan right = new GroupPlan(new Group(idGenerator.getNextId(), new LogicalProperties(
+ new Supplier<List<Slot>>() {
+ @Override
+ public List<Slot> get() {
+ return Lists.newArrayList(b, c);
+ }
+ })));
+ LogicalJoin join = new LogicalJoin(JoinType.LEFT_OUTER_JOIN, Lists.newArrayList(eq),
+ left, right);
+ Statistics outputStats = JoinEstimation.estimate(leftStats, rightStats, join);
+ ColumnStatistic outAStats = outputStats.findColumnStatistics(a);
+ Assertions.assertNotNull(outAStats);
+ Assertions.assertEquals(10, outAStats.ndv);
+ ColumnStatistic outBStats = outputStats.findColumnStatistics(b);
+ Assertions.assertNotNull(outAStats);
+ Assertions.assertEquals(0, outBStats.ndv);
+ ColumnStatistic outCStats = outputStats.findColumnStatistics(c);
+ Assertions.assertNotNull(outAStats);
+ Assertions.assertEquals(20.0, outCStats.ndv);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org