You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/01/22 12:31:29 UTC
spark git commit: [SPARK-23170][SQL] Dump the statistics of effective
runs of analyzer and optimizer rules
Repository: spark
Updated Branches:
refs/heads/master 73281161f -> 78801881c
[SPARK-23170][SQL] Dump the statistics of effective runs of analyzer and optimizer rules
## What changes were proposed in this pull request?
Dump the statistics of effective runs of analyzer and optimizer rules.
## How was this patch tested?
Do a manual run of TPCDSQuerySuite
```
=== Metrics of Analyzer/Optimizer Rules ===
Total number of runs: 175899
Total time: 25.486559948 seconds
Rule Effective Time / Total Time Effective Runs / Total Runs
org.apache.spark.sql.catalyst.optimizer.ColumnPruning 1603280450 / 2868461549 761 / 1877
org.apache.spark.sql.catalyst.analysis.Analyzer$CTESubstitution 2045860009 / 2056602674 37 / 788
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAggregateFunctions 440719059 / 1693110949 38 / 1982
org.apache.spark.sql.catalyst.optimizer.Optimizer$OptimizeSubqueries 1429834919 / 1446016225 39 / 285
org.apache.spark.sql.catalyst.optimizer.PruneFilters 33273083 / 1389586938 3 / 1592
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences 821183615 / 1266668754 616 / 1982
org.apache.spark.sql.catalyst.optimizer.ReorderJoin 775837028 / 866238225 132 / 1592
org.apache.spark.sql.catalyst.analysis.DecimalPrecision 550683593 / 748854507 211 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveSubquery 513075345 / 634370596 49 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$FixNullability 33475731 / 606406532 12 / 742
org.apache.spark.sql.catalyst.analysis.TypeCoercion$ImplicitTypeCasts 193144298 / 545403925 86 / 1982
org.apache.spark.sql.catalyst.optimizer.BooleanSimplification 18651497 / 495725004 7 / 1592
org.apache.spark.sql.catalyst.optimizer.PushPredicateThroughJoin 369257217 / 489934378 709 / 1592
org.apache.spark.sql.catalyst.optimizer.RemoveRedundantAliases 3707000 / 468291609 9 / 1592
org.apache.spark.sql.catalyst.optimizer.InferFiltersFromConstraints 410155900 / 435254175 192 / 285
org.apache.spark.sql.execution.datasources.FindDataSourceTable 348885539 / 371855866 233 / 1982
org.apache.spark.sql.catalyst.optimizer.NullPropagation 11307645 / 307531225 26 / 1592
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions 120324545 / 304948785 294 / 1982
org.apache.spark.sql.catalyst.analysis.TypeCoercion$FunctionArgumentConversion 92323199 / 286695007 38 / 1982
org.apache.spark.sql.catalyst.optimizer.PushDownPredicate 230084193 / 265845972 785 / 1592
org.apache.spark.sql.catalyst.analysis.TypeCoercion$PromoteStrings 45938401 / 265144009 40 / 1982
org.apache.spark.sql.catalyst.analysis.TypeCoercion$InConversion 14888776 / 261499450 1 / 1982
org.apache.spark.sql.catalyst.analysis.TypeCoercion$CaseWhenCoercion 113796384 / 244913861 29 / 1982
org.apache.spark.sql.catalyst.optimizer.ConstantFolding 65008069 / 236548480 126 / 1592
org.apache.spark.sql.catalyst.analysis.Analyzer$ExtractGenerator 0 / 226338929 0 / 1982
org.apache.spark.sql.catalyst.analysis.ResolveTimeZone 98134906 / 221323770 417 / 1982
org.apache.spark.sql.catalyst.optimizer.ReorderAssociativeOperator 0 / 208421703 0 / 1592
org.apache.spark.sql.catalyst.optimizer.OptimizeIn 8762534 / 199351958 16 / 1592
org.apache.spark.sql.catalyst.analysis.TypeCoercion$DateTimeOperations 11980016 / 190779046 27 / 1982
org.apache.spark.sql.catalyst.optimizer.SimplifyBinaryComparison 0 / 188887385 0 / 1592
org.apache.spark.sql.catalyst.optimizer.SimplifyConditionals 0 / 186812106 0 / 1592
org.apache.spark.sql.catalyst.optimizer.SimplifyCaseConversionExpressions 0 / 183885230 0 / 1592
org.apache.spark.sql.catalyst.optimizer.SimplifyCasts 17128295 / 182901910 69 / 1592
org.apache.spark.sql.catalyst.analysis.TypeCoercion$Division 14579110 / 180309340 8 / 1982
org.apache.spark.sql.catalyst.analysis.TypeCoercion$BooleanEquality 0 / 176740516 0 / 1982
org.apache.spark.sql.catalyst.analysis.TypeCoercion$IfCoercion 0 / 170781986 0 / 1982
org.apache.spark.sql.catalyst.optimizer.LikeSimplification 771605 / 164136736 1 / 1592
org.apache.spark.sql.catalyst.optimizer.RemoveDispensableExpressions 0 / 155958962 0 / 1592
org.apache.spark.sql.catalyst.analysis.ResolveCreateNamedStruct 0 / 151222943 0 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveWindowOrder 7534632 / 146596355 14 / 1982
org.apache.spark.sql.catalyst.analysis.TypeCoercion$EltCoercion 0 / 144488654 0 / 1982
org.apache.spark.sql.catalyst.analysis.TypeCoercion$ConcatCoercion 0 / 142403338 0 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveWindowFrame 12067635 / 141500665 21 / 1982
org.apache.spark.sql.catalyst.analysis.TimeWindowing 0 / 140431958 0 / 1982
org.apache.spark.sql.catalyst.analysis.TypeCoercion$WindowFrameCoercion 0 / 125471960 0 / 1982
org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin 14226972 / 124922019 11 / 1592
org.apache.spark.sql.catalyst.analysis.TypeCoercion$StackCoercion 0 / 123613887 0 / 1982
org.apache.spark.sql.catalyst.optimizer.RewriteCorrelatedScalarSubquery 8491071 / 121179056 7 / 1592
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveGroupingAnalytics 55526073 / 120290529 11 / 1982
org.apache.spark.sql.catalyst.optimizer.ConstantPropagation 0 / 113886790 0 / 1592
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveDeserializer 52383759 / 107160222 148 / 1982
org.apache.spark.sql.catalyst.analysis.CleanupAliases 52543524 / 102091518 344 / 1086
org.apache.spark.sql.catalyst.optimizer.RemoveRedundantProject 40682895 / 94403652 342 / 1877
org.apache.spark.sql.catalyst.analysis.Analyzer$ExtractWindowExpressions 38473816 / 89740578 23 / 1982
org.apache.spark.sql.catalyst.optimizer.CollapseProject 46806090 / 83315506 281 / 1877
org.apache.spark.sql.catalyst.optimizer.FoldablePropagation 0 / 78750087 0 / 1592
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases 13742765 / 77227258 47 / 1982
org.apache.spark.sql.catalyst.optimizer.CombineFilters 53386729 / 76960344 448 / 1592
org.apache.spark.sql.execution.datasources.DataSourceAnalysis 68034341 / 75724186 24 / 742
org.apache.spark.sql.catalyst.analysis.Analyzer$LookupFunctions 0 / 71151084 0 / 750
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveMissingReferences 12139848 / 67599140 8 / 1982
org.apache.spark.sql.catalyst.optimizer.PullupCorrelatedPredicates 45017938 / 65968777 23 / 285
org.apache.spark.sql.execution.datasources.v2.PushDownOperatorsToDataSource 0 / 60937767 0 / 285
org.apache.spark.sql.catalyst.optimizer.CollapseRepartition 0 / 59897237 0 / 1592
org.apache.spark.sql.catalyst.optimizer.PushProjectionThroughUnion 8547262 / 53941370 10 / 1592
org.apache.spark.sql.catalyst.analysis.Analyzer$HandleNullInputsForUDF 0 / 52735976 0 / 742
org.apache.spark.sql.catalyst.analysis.TypeCoercion$WidenSetOperationTypes 9797713 / 52401665 9 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$PullOutNondeterministic 0 / 51741500 0 / 742
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveRelations 28614911 / 51061186 233 / 1990
org.apache.spark.sql.execution.datasources.PruneFileSourcePartitions 0 / 50621510 0 / 285
org.apache.spark.sql.catalyst.optimizer.CombineUnions 2777800 / 50262112 17 / 1877
org.apache.spark.sql.catalyst.analysis.Analyzer$GlobalAggregates 1640641 / 49633909 46 / 1982
org.apache.spark.sql.catalyst.optimizer.DecimalAggregates 20198374 / 48488419 100 / 385
org.apache.spark.sql.catalyst.optimizer.LimitPushDown 0 / 45052523 0 / 1592
org.apache.spark.sql.catalyst.optimizer.CombineLimits 0 / 44719443 0 / 1592
org.apache.spark.sql.catalyst.optimizer.EliminateSorts 0 / 44216930 0 / 1592
org.apache.spark.sql.catalyst.optimizer.RewritePredicateSubquery 36235699 / 44165786 148 / 285
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNewInstance 0 / 42750307 0 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveUpCast 0 / 41811748 0 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveOrdinalInOrderByAndGroupBy 3819476 / 41776562 4 / 1982
org.apache.spark.sql.catalyst.optimizer.ComputeCurrentTime 0 / 40527808 0 / 285
org.apache.spark.sql.catalyst.optimizer.CollapseWindow 0 / 36832538 0 / 1592
org.apache.spark.sql.catalyst.optimizer.EliminateSerialization 0 / 36120667 0 / 1592
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAggAliasInGroupBy 0 / 32435826 0 / 1982
org.apache.spark.sql.execution.datasources.PreprocessTableCreation 0 / 32145218 0 / 742
org.apache.spark.sql.execution.datasources.ResolveSQLOnFile 0 / 30295614 0 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolvePivot 0 / 30111655 0 / 1982
org.apache.spark.sql.catalyst.expressions.codegen.package$ExpressionCanonicalizer$CleanExpressions 59930 / 28038201 26 / 8280
org.apache.spark.sql.catalyst.analysis.ResolveInlineTables 0 / 27808108 0 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveSubqueryColumnAliases 0 / 27066690 0 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveGenerate 0 / 26660210 0 / 1982
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveNaturalAndUsingJoin 0 / 25255184 0 / 1982
org.apache.spark.sql.catalyst.analysis.ResolveTableValuedFunctions 0 / 24663088 0 / 1990
org.apache.spark.sql.catalyst.analysis.SubstituteUnresolvedOrdinals 9709079 / 24450670 4 / 788
org.apache.spark.sql.catalyst.analysis.ResolveHints$ResolveBroadcastHints 0 / 23776535 0 / 750
org.apache.spark.sql.catalyst.optimizer.ReplaceExpressions 0 / 22697895 0 / 285
org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts 0 / 22523798 0 / 285
org.apache.spark.sql.catalyst.optimizer.ReplaceDistinctWithAggregate 988593 / 21535410 15 / 300
org.apache.spark.sql.catalyst.optimizer.EliminateMapObjects 0 / 20269996 0 / 285
org.apache.spark.sql.catalyst.optimizer.RewriteDistinctAggregates 0 / 19388592 0 / 285
org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases 17675532 / 18971185 215 / 285
org.apache.spark.sql.catalyst.optimizer.GetCurrentDatabase 0 / 18271152 0 / 285
org.apache.spark.sql.catalyst.optimizer.PropagateEmptyRelation 2077097 / 17190855 3 / 288
org.apache.spark.sql.catalyst.analysis.EliminateBarriers 0 / 16736359 0 / 1086
org.apache.spark.sql.execution.OptimizeMetadataOnlyQuery 0 / 16669341 0 / 285
org.apache.spark.sql.catalyst.analysis.UpdateOuterReferences 0 / 14470235 0 / 742
org.apache.spark.sql.catalyst.optimizer.ReplaceExceptWithAntiJoin 6715625 / 12190561 1 / 300
org.apache.spark.sql.catalyst.optimizer.ReplaceIntersectWithSemiJoin 3451793 / 11431432 7 / 300
org.apache.spark.sql.execution.python.ExtractPythonUDFFromAggregate 0 / 10810568 0 / 285
org.apache.spark.sql.catalyst.optimizer.RemoveRepetitionFromGroupExpressions 344198 / 10475276 1 / 286
org.apache.spark.sql.catalyst.analysis.Analyzer$WindowsSubstitution 0 / 10386630 0 / 788
org.apache.spark.sql.catalyst.analysis.EliminateUnions 0 / 10096526 0 / 788
org.apache.spark.sql.catalyst.analysis.AliasViewChild 0 / 9991706 0 / 742
org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation 0 / 9649334 0 / 288
org.apache.spark.sql.catalyst.analysis.ResolveHints$RemoveAllHints 0 / 8739109 0 / 750
org.apache.spark.sql.execution.datasources.PreprocessTableInsertion 0 / 8420889 0 / 742
org.apache.spark.sql.catalyst.analysis.EliminateView 0 / 8319134 0 / 285
org.apache.spark.sql.catalyst.optimizer.RemoveLiteralFromGroupExpressions 0 / 7392627 0 / 286
org.apache.spark.sql.catalyst.optimizer.ReplaceExceptWithFilter 0 / 7170516 0 / 300
org.apache.spark.sql.catalyst.optimizer.SimplifyCreateArrayOps 0 / 7109643 0 / 1592
org.apache.spark.sql.catalyst.optimizer.SimplifyCreateStructOps 0 / 6837590 0 / 1592
org.apache.spark.sql.catalyst.optimizer.SimplifyCreateMapOps 0 / 6617848 0 / 1592
org.apache.spark.sql.catalyst.optimizer.CombineConcats 0 / 5768406 0 / 1592
org.apache.spark.sql.catalyst.optimizer.ReplaceDeduplicateWithAggregate 0 / 5349831 0 / 285
org.apache.spark.sql.catalyst.optimizer.CombineTypedFilters 0 / 5186642 0 / 285
org.apache.spark.sql.catalyst.optimizer.EliminateDistinct 0 / 2427686 0 / 285
org.apache.spark.sql.catalyst.optimizer.CostBasedJoinReorder 0 / 2420436 0 / 285
```
Author: gatorsmile <ga...@gmail.com>
Closes #20342 from gatorsmile/reportExecution.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/78801881
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/78801881
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/78801881
Branch: refs/heads/master
Commit: 78801881c405de47f7e53eea3e0420dd69593dbd
Parents: 7328116
Author: gatorsmile <ga...@gmail.com>
Authored: Mon Jan 22 04:31:24 2018 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Jan 22 04:31:24 2018 -0800
----------------------------------------------------------------------
.../catalyst/rules/QueryExecutionMetering.scala | 91 ++++++++++++++++++++
.../spark/sql/catalyst/rules/RuleExecutor.scala | 32 ++++---
.../apache/spark/sql/BenchmarkQueryTest.scala | 2 +-
.../apache/spark/sql/SQLQueryTestSuite.scala | 2 +-
.../hive/execution/HiveCompatibilitySuite.scala | 2 +-
5 files changed, 109 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/78801881/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
new file mode 100644
index 0000000..62f7541
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.rules
+
+import scala.collection.JavaConverters._
+
+import com.google.common.util.concurrent.AtomicLongMap
+
+case class QueryExecutionMetering() {
+ private val timeMap = AtomicLongMap.create[String]()
+ private val numRunsMap = AtomicLongMap.create[String]()
+ private val numEffectiveRunsMap = AtomicLongMap.create[String]()
+ private val timeEffectiveRunsMap = AtomicLongMap.create[String]()
+
+ /** Resets statistics about time spent running specific rules */
+ def resetMetrics(): Unit = {
+ timeMap.clear()
+ numRunsMap.clear()
+ numEffectiveRunsMap.clear()
+ timeEffectiveRunsMap.clear()
+ }
+
+ def totalTime: Long = {
+ timeMap.sum()
+ }
+
+ def totalNumRuns: Long = {
+ numRunsMap.sum()
+ }
+
+ def incExecutionTimeBy(ruleName: String, delta: Long): Unit = {
+ timeMap.addAndGet(ruleName, delta)
+ }
+
+ def incTimeEffectiveExecutionBy(ruleName: String, delta: Long): Unit = {
+ timeEffectiveRunsMap.addAndGet(ruleName, delta)
+ }
+
+ def incNumEffectiveExecution(ruleName: String): Unit = {
+ numEffectiveRunsMap.incrementAndGet(ruleName)
+ }
+
+ def incNumExecution(ruleName: String): Unit = {
+ numRunsMap.incrementAndGet(ruleName)
+ }
+
+ /** Dump statistics about time spent running specific rules. */
+ def dumpTimeSpent(): String = {
+ val map = timeMap.asMap().asScala
+ val maxLengthRuleNames = map.keys.map(_.toString.length).max
+
+ val colRuleName = "Rule".padTo(maxLengthRuleNames, " ").mkString
+ val colRunTime = "Effective Time / Total Time".padTo(len = 47, " ").mkString
+ val colNumRuns = "Effective Runs / Total Runs".padTo(len = 47, " ").mkString
+
+ val ruleMetrics = map.toSeq.sortBy(_._2).reverseMap { case (name, time) =>
+ val timeEffectiveRun = timeEffectiveRunsMap.get(name)
+ val numRuns = numRunsMap.get(name)
+ val numEffectiveRun = numEffectiveRunsMap.get(name)
+
+ val ruleName = name.padTo(maxLengthRuleNames, " ").mkString
+ val runtimeValue = s"$timeEffectiveRun / $time".padTo(len = 47, " ").mkString
+ val numRunValue = s"$numEffectiveRun / $numRuns".padTo(len = 47, " ").mkString
+ s"$ruleName $runtimeValue $numRunValue"
+ }.mkString("\n", "\n", "")
+
+ s"""
+ |=== Metrics of Analyzer/Optimizer Rules ===
+ |Total number of runs: $totalNumRuns
+ |Total time: ${totalTime / 1000000000D} seconds
+ |
+ |$colRuleName $colRunTime $colNumRuns
+ |$ruleMetrics
+ """.stripMargin
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/78801881/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
index 7e4b784..dccb44d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
@@ -17,10 +17,6 @@
package org.apache.spark.sql.catalyst.rules
-import scala.collection.JavaConverters._
-
-import com.google.common.util.concurrent.AtomicLongMap
-
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.errors.TreeNodeException
import org.apache.spark.sql.catalyst.trees.TreeNode
@@ -28,18 +24,16 @@ import org.apache.spark.sql.catalyst.util.sideBySide
import org.apache.spark.util.Utils
object RuleExecutor {
- protected val timeMap = AtomicLongMap.create[String]()
-
- /** Resets statistics about time spent running specific rules */
- def resetTime(): Unit = timeMap.clear()
+ protected val queryExecutionMeter = QueryExecutionMetering()
/** Dump statistics about time spent running specific rules. */
def dumpTimeSpent(): String = {
- val map = timeMap.asMap().asScala
- val maxSize = map.keys.map(_.toString.length).max
- map.toSeq.sortBy(_._2).reverseMap { case (k, v) =>
- s"${k.padTo(maxSize, " ").mkString} $v"
- }.mkString("\n", "\n", "")
+ queryExecutionMeter.dumpTimeSpent()
+ }
+
+ /** Resets statistics about time spent running specific rules */
+ def resetMetrics(): Unit = {
+ queryExecutionMeter.resetMetrics()
}
}
@@ -77,6 +71,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
*/
def execute(plan: TreeType): TreeType = {
var curPlan = plan
+ val queryExecutionMetrics = RuleExecutor.queryExecutionMeter
batches.foreach { batch =>
val batchStartPlan = curPlan
@@ -91,15 +86,18 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
val startTime = System.nanoTime()
val result = rule(plan)
val runTime = System.nanoTime() - startTime
- RuleExecutor.timeMap.addAndGet(rule.ruleName, runTime)
if (!result.fastEquals(plan)) {
+ queryExecutionMetrics.incNumEffectiveExecution(rule.ruleName)
+ queryExecutionMetrics.incTimeEffectiveExecutionBy(rule.ruleName, runTime)
logTrace(
s"""
|=== Applying Rule ${rule.ruleName} ===
|${sideBySide(plan.treeString, result.treeString).mkString("\n")}
""".stripMargin)
}
+ queryExecutionMetrics.incExecutionTimeBy(rule.ruleName, runTime)
+ queryExecutionMetrics.incNumExecution(rule.ruleName)
// Run the structural integrity checker against the plan after each rule.
if (!isPlanIntegral(result)) {
@@ -135,9 +133,9 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
if (!batchStartPlan.fastEquals(curPlan)) {
logDebug(
s"""
- |=== Result of Batch ${batch.name} ===
- |${sideBySide(batchStartPlan.treeString, curPlan.treeString).mkString("\n")}
- """.stripMargin)
+ |=== Result of Batch ${batch.name} ===
+ |${sideBySide(batchStartPlan.treeString, curPlan.treeString).mkString("\n")}
+ """.stripMargin)
} else {
logTrace(s"Batch ${batch.name} has no effect.")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/78801881/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
index 7037749..e51aad0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
@@ -46,7 +46,7 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSQLContext with B
override def beforeAll() {
super.beforeAll()
- RuleExecutor.resetTime()
+ RuleExecutor.resetMetrics()
}
protected def checkGeneratedCode(plan: SparkPlan): Unit = {
http://git-wip-us.apache.org/repos/asf/spark/blob/78801881/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index e3901af..054ada5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -291,7 +291,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext {
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
// Add Locale setting
Locale.setDefault(Locale.US)
- RuleExecutor.resetTime()
+ RuleExecutor.resetMetrics()
}
override def afterAll(): Unit = {
http://git-wip-us.apache.org/repos/asf/spark/blob/78801881/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index 45791c6..cebaad5 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -62,7 +62,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
// Fix session local timezone to America/Los_Angeles for those timezone sensitive tests
// (timestamp_*)
TestHive.setConf(SQLConf.SESSION_LOCAL_TIMEZONE, "America/Los_Angeles")
- RuleExecutor.resetTime()
+ RuleExecutor.resetMetrics()
}
override def afterAll() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org