You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by vi...@apache.org on 2024/08/28 20:52:52 UTC
(datafusion-comet) branch main updated: chore: Add spilling metrics of SortMergeJoin (#878)
This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new f4400f5b chore: Add spilling metrics of SortMergeJoin (#878)
f4400f5b is described below
commit f4400f5ba6a62002dd8eefa0ff386d5cca49c410
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Wed Aug 28 13:52:47 2024 -0700
chore: Add spilling metrics of SortMergeJoin (#878)
---
.../org/apache/spark/shuffle/sort/RowPartition.scala | 6 +++++-
.../org/apache/spark/sql/comet/CometMetricNode.scala | 19 +++++++++++++++++++
.../scala/org/apache/spark/sql/comet/operators.scala | 9 +--------
.../scala/org/apache/comet/exec/CometExecSuite.scala | 2 ++
4 files changed, 27 insertions(+), 9 deletions(-)
diff --git a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala
index 32d64fad..d3536850 100644
--- a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala
+++ b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala
@@ -30,7 +30,11 @@ class RowPartition(initialSize: Int) {
rowSizes += size
}
- def getNumRows: Int = rowAddresses.size
+ def getNumRows: Int = if (rowAddresses == null) {
+ 0
+ } else {
+ rowAddresses.size
+ }
def getRowAddresses: Array[Long] = {
val array = rowAddresses.toArray
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala
index 7288455f..195a0fde 100644
--- a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala
+++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala
@@ -101,6 +101,25 @@ object CometMetricNode {
"join_time" -> SQLMetrics.createNanoTimingMetric(sc, "Total time for joining"))
}
+ /**
+ * SQL Metrics for DataFusion SortMergeJoin
+ */
+ def sortMergeJoinMetrics(sc: SparkContext): Map[String, SQLMetric] = {
+ Map(
+ "peak_mem_used" ->
+ SQLMetrics.createSizeMetric(sc, "Memory used by build-side"),
+ "input_batches" ->
+ SQLMetrics.createMetric(sc, "Number of batches consumed by probe-side"),
+ "input_rows" ->
+ SQLMetrics.createMetric(sc, "Number of rows consumed by probe-side"),
+ "output_batches" -> SQLMetrics.createMetric(sc, "Number of batches produced"),
+ "output_rows" -> SQLMetrics.createMetric(sc, "Number of rows produced"),
+ "join_time" -> SQLMetrics.createNanoTimingMetric(sc, "Total time for joining"),
+ "spill_count" -> SQLMetrics.createMetric(sc, "Count of spills"),
+ "spilled_bytes" -> SQLMetrics.createSizeMetric(sc, "Total spilled bytes"),
+ "spilled_rows" -> SQLMetrics.createMetric(sc, "Total spilled rows"))
+ }
+
/**
* Creates a [[CometMetricNode]] from a [[CometPlan]].
*/
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala
index 7c335fbb..546d15c4 100644
--- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala
+++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala
@@ -933,14 +933,7 @@ case class CometSortMergeJoinExec(
Objects.hashCode(leftKeys, rightKeys, condition, left, right)
override lazy val metrics: Map[String, SQLMetric] =
- Map(
- "input_batches" -> SQLMetrics.createMetric(sparkContext, "Number of batches consumed"),
- "input_rows" -> SQLMetrics.createMetric(sparkContext, "Number of rows consumed"),
- "output_batches" -> SQLMetrics.createMetric(sparkContext, "Number of batches produced"),
- "output_rows" -> SQLMetrics.createMetric(sparkContext, "Number of rows produced"),
- "peak_mem_used" ->
- SQLMetrics.createSizeMetric(sparkContext, "Peak memory used for buffered data"),
- "join_time" -> SQLMetrics.createNanoTimingMetric(sparkContext, "Total time for joining"))
+ CometMetricNode.sortMergeJoinMetrics(sparkContext)
}
case class CometScanWrapper(override val nativeOp: Operator, override val originalPlan: SparkPlan)
diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
index bfddd74d..b632d5f8 100644
--- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
@@ -519,6 +519,8 @@ class CometExecSuite extends CometTestBase {
assert(metrics("peak_mem_used").value > 1L)
assert(metrics.contains("join_time"))
assert(metrics("join_time").value > 1L)
+ assert(metrics.contains("spill_count"))
+ assert(metrics("spill_count").value == 0)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org