You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ul...@apache.org on 2022/02/10 01:44:02 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #1884] Fix compile SQL measurement
This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 33eda21 [KYUUBI #1884] Fix compile SQL measurement
33eda21 is described below
commit 33eda2159fbd9b16ffd4e876d898945673ea4b91
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Thu Feb 10 09:43:52 2022 +0800
[KYUUBI #1884] Fix compile SQL measurement
### _Why are the changes needed?_
The previous measurement of parsing and analysis does not make sense because `Dataset` calls `queryExecution.assertAnalyzed()` in the constructor.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #1884 from pan3793/measure.
Closes #1884
8da93874 [Cheng Pan] Fix compile SQL measurement
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: ulysses-you <ul...@apache.org>
---
.../org/apache/kyuubi/tpcds/benchmark/Query.scala | 20 +++++++-------------
1 file changed, 7 insertions(+), 13 deletions(-)
diff --git a/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/Query.scala b/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/Query.scala
index 5eba308..e3e82bb 100644
--- a/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/Query.scala
+++ b/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/Query.scala
@@ -22,6 +22,7 @@ import scala.collection.mutable.ArrayBuffer
import scala.language.implicitConversions
import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.catalyst.QueryPlanningTracker
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
import org.apache.spark.sql.execution.SparkPlan
@@ -66,19 +67,12 @@ class Query(
try {
val dataFrame = buildDataFrame
val queryExecution = dataFrame.queryExecution
- // We are not counting the time of ScalaReflection.convertRowToScala.
- val parsingTime = measureTimeMs {
- queryExecution.logical
- }
- val analysisTime = measureTimeMs {
- queryExecution.analyzed
- }
- val optimizationTime = measureTimeMs {
- queryExecution.optimizedPlan
- }
- val planningTime = measureTimeMs {
- queryExecution.executedPlan
- }
+ queryExecution.executedPlan
+ val phases = queryExecution.tracker.phases
+ val parsingTime = phases(QueryPlanningTracker.PARSING).durationMs.toDouble
+ val analysisTime = phases(QueryPlanningTracker.ANALYSIS).durationMs.toDouble
+ val optimizationTime = phases(QueryPlanningTracker.OPTIMIZATION).durationMs.toDouble
+ val planningTime = phases(QueryPlanningTracker.PLANNING).durationMs.toDouble
val breakdownResults =
if (includeBreakdown) {