You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/07 07:33:12 UTC
[doris] branch master updated: [fix](Nereids) extract complicated time string in evaluating cost model framework (#17864)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 63994e351f [fix](Nereids) extract complicated time string in evaluating cost model framework (#17864)
63994e351f is described below
commit 63994e351fb41cf0e1bca571c5b0afc4773c167a
Author: 谢健 <ji...@gmail.com>
AuthorDate: Fri Apr 7 15:33:04 2023 +0800
[fix](Nereids) extract complicated time string in evaluating cost model framework (#17864)
1. The time string in the profile can be "xx s xx ms". The framework should extract time with re package to support more complicated time string
2. Add stats for sortNode and AggNode in `withChildren`
---
.../plans/physical/PhysicalHashAggregate.java | 4 +++-
.../trees/plans/physical/PhysicalQuickSort.java | 3 ++-
.../physical/PhysicalStorageLayerAggregate.java | 4 +++-
tools/cost_model_evaluate/sql_executor.py | 24 +++++++++++++++++++++-
4 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
index 79d13ccfc3..d1df4391ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
@@ -228,7 +228,9 @@ public class PhysicalHashAggregate<CHILD_TYPE extends Plan> extends PhysicalUnar
public PhysicalHashAggregate<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
return new PhysicalHashAggregate<>(groupByExpressions, outputExpressions, partitionExpressions,
- aggregateParam, maybeUsingStream, getLogicalProperties(), requireProperties, children.get(0));
+ aggregateParam, maybeUsingStream, groupExpression, getLogicalProperties(),
+ requireProperties, physicalProperties, statistics,
+ children.get(0));
}
public PhysicalHashAggregate<CHILD_TYPE> withPartitionExpressions(List<Expression> partitionExpressions) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java
index aacefd1fcb..cb2e3118d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java
@@ -75,7 +75,8 @@ public class PhysicalQuickSort<CHILD_TYPE extends Plan> extends AbstractPhysical
@Override
public PhysicalQuickSort<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
- return new PhysicalQuickSort<>(orderKeys, phase, getLogicalProperties(), children.get(0));
+ return new PhysicalQuickSort<>(orderKeys, phase, groupExpression, getLogicalProperties(), physicalProperties,
+ statistics, children.get(0));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java
index 21f1df9b99..e6e2ca9b9c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java
@@ -131,7 +131,9 @@ public class PhysicalStorageLayerAggregate extends PhysicalRelation {
@Override
public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
Statistics statistics) {
- return new PhysicalStorageLayerAggregate(relation, aggOp, groupExpression,
+ return new PhysicalStorageLayerAggregate(
+ (PhysicalRelation) relation.withPhysicalPropertiesAndStats(null, statistics),
+ aggOp, groupExpression,
getLogicalProperties(), physicalProperties, statistics);
}
diff --git a/tools/cost_model_evaluate/sql_executor.py b/tools/cost_model_evaluate/sql_executor.py
index 511e12c8ad..c38cc322db 100644
--- a/tools/cost_model_evaluate/sql_executor.py
+++ b/tools/cost_model_evaluate/sql_executor.py
@@ -18,6 +18,7 @@
from unittest import result
import mysql.connector
from typing import List, Tuple
+import re
class SQLExecutor:
@@ -43,7 +44,28 @@ class SQLExecutor:
def get_execute_time(self, query: str) -> float:
self.execute_query(query, None)
profile = self.execute_query("show query profile\"\"", None)
- return float(profile[0][self.wait_fetch_time_index].replace("ms", ""))
+ return self.get_n_ms(profile[0][self.wait_fetch_time_index])
+
+ def get_n_ms(self, t: str):
+ res = re.search(r"(\d+h)*(\d+min)*(\d+s)*(\d+ms)", t)
+ if res is None:
+ raise Exception(f"invalid time {t}")
+ n = 0
+
+ h = res.group(1)
+ if h is not None:
+ n += int(h.replace("h", "")) * 60 * 60 * 1000
+ min = res.group(2)
+ if min is not None != 0:
+ n += int(min.replace("min", "")) * 60 * 1000
+ s = res.group(3)
+ if s is not None != 0:
+ n += int(s.replace("s", "")) * 1000
+ ms = res.group(4)
+ if len(ms) != 0:
+ n += int(ms.replace("ms", ""))
+
+ return n
def execute_many_queries(self, queries: List[Tuple[str, Tuple]]) -> List[List[Tuple]]:
results = []
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org