You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yu...@apache.org on 2020/09/05 15:58:41 UTC
[spark] branch branch-3.0 updated: [SPARK-32791][SQL]
Non-partitioned table metric should not have dynamic partition pruning time
This is an automated email from the ASF dual-hosted git repository.
yumwang pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 0c7616b [SPARK-32791][SQL] Non-partitioned table metric should not have dynamic partition pruning time
0c7616b is described below
commit 0c7616b9b5c5f2d69b0410b988bf5e3955e34a6c
Author: Yuming Wang <yu...@ebay.com>
AuthorDate: Sat Sep 5 23:49:17 2020 +0800
[SPARK-32791][SQL] Non-partitioned table metric should not have dynamic partition pruning time
### What changes were proposed in this pull request?
This pr make non-partitioned table metric should not have dynamic partition pruning time.
### Why are the changes needed?
It is useless for non-partitioned table.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manual test
Before this pr:
After this pr:
Closes #29641 from wangyum/SPARK-32791.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: Yuming Wang <yu...@ebay.com>
(cherry picked from commit 0b3bb45b893005d73afb15e4904ac6440c87811c)
Signed-off-by: Yuming Wang <yu...@ebay.com>
---
.../org/apache/spark/sql/execution/DataSourceScanExec.scala | 11 ++++++-----
.../org/apache/spark/sql/DynamicPartitionPruningSuite.scala | 3 +++
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 2147abd..447e0a6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -438,9 +438,7 @@ case class FileSourceScanExec(
"numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
"numFiles" -> SQLMetrics.createMetric(sparkContext, "number of files read"),
"metadataTime" -> SQLMetrics.createTimingMetric(sparkContext, "metadata time"),
- "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read"),
- "pruningTime" ->
- SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time")
+ "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read")
) ++ {
// Tracking scan time has overhead, we can't afford to do it for each row, and can only do
// it for each batch.
@@ -451,9 +449,12 @@ case class FileSourceScanExec(
}
} ++ {
if (relation.partitionSchemaOption.isDefined) {
- Some("numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"))
+ Map(
+ "numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"),
+ "pruningTime" ->
+ SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time"))
} else {
- None
+ Map.empty[String, SQLMetric]
}
} ++ staticMetrics
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index 8486d1d..9416331 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -345,6 +345,7 @@ abstract class DynamicPartitionPruningSuiteBase
val allFilesNum = scan1.metrics("numFiles").value
val allFilesSize = scan1.metrics("filesSize").value
assert(scan1.metrics("numPartitions").value === numPartitions)
+ assert(scan1.metrics("pruningTime").value === -1)
// No dynamic partition pruning, so no static metrics
// Only files from fid = 5 partition are scanned
@@ -358,6 +359,7 @@ abstract class DynamicPartitionPruningSuiteBase
assert(0 < partFilesNum && partFilesNum < allFilesNum)
assert(0 < partFilesSize && partFilesSize < allFilesSize)
assert(scan2.metrics("numPartitions").value === 1)
+ assert(scan2.metrics("pruningTime").value === -1)
// Dynamic partition pruning is used
// Static metrics are as-if reading the whole fact table
@@ -370,6 +372,7 @@ abstract class DynamicPartitionPruningSuiteBase
assert(scan3.metrics("numFiles").value == partFilesNum)
assert(scan3.metrics("filesSize").value == partFilesSize)
assert(scan3.metrics("numPartitions").value === 1)
+ assert(scan3.metrics("pruningTime").value !== -1)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org