You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by vi...@apache.org on 2024/08/28 20:52:52 UTC

(datafusion-comet) branch main updated: chore: Add spilling metrics of SortMergeJoin (#878)

This is an automated email from the ASF dual-hosted git repository.

viirya pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


The following commit(s) were added to refs/heads/main by this push:
     new f4400f5b chore: Add spilling metrics of SortMergeJoin (#878)
f4400f5b is described below

commit f4400f5ba6a62002dd8eefa0ff386d5cca49c410
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Wed Aug 28 13:52:47 2024 -0700

    chore: Add spilling metrics of SortMergeJoin (#878)
---
 .../org/apache/spark/shuffle/sort/RowPartition.scala  |  6 +++++-
 .../org/apache/spark/sql/comet/CometMetricNode.scala  | 19 +++++++++++++++++++
 .../scala/org/apache/spark/sql/comet/operators.scala  |  9 +--------
 .../scala/org/apache/comet/exec/CometExecSuite.scala  |  2 ++
 4 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala
index 32d64fad..d3536850 100644
--- a/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala
+++ b/spark/src/main/scala/org/apache/spark/shuffle/sort/RowPartition.scala
@@ -30,7 +30,11 @@ class RowPartition(initialSize: Int) {
     rowSizes += size
   }
 
-  def getNumRows: Int = rowAddresses.size
+  def getNumRows: Int = if (rowAddresses == null) {
+    0
+  } else {
+    rowAddresses.size
+  }
 
   def getRowAddresses: Array[Long] = {
     val array = rowAddresses.toArray
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala
index 7288455f..195a0fde 100644
--- a/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala
+++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala
@@ -101,6 +101,25 @@ object CometMetricNode {
       "join_time" -> SQLMetrics.createNanoTimingMetric(sc, "Total time for joining"))
   }
 
+  /**
+   * SQL Metrics for DataFusion SortMergeJoin
+   */
+  def sortMergeJoinMetrics(sc: SparkContext): Map[String, SQLMetric] = {
+    Map(
+      "peak_mem_used" ->
+        SQLMetrics.createSizeMetric(sc, "Memory used by build-side"),
+      "input_batches" ->
+        SQLMetrics.createMetric(sc, "Number of batches consumed by probe-side"),
+      "input_rows" ->
+        SQLMetrics.createMetric(sc, "Number of rows consumed by probe-side"),
+      "output_batches" -> SQLMetrics.createMetric(sc, "Number of batches produced"),
+      "output_rows" -> SQLMetrics.createMetric(sc, "Number of rows produced"),
+      "join_time" -> SQLMetrics.createNanoTimingMetric(sc, "Total time for joining"),
+      "spill_count" -> SQLMetrics.createMetric(sc, "Count of spills"),
+      "spilled_bytes" -> SQLMetrics.createSizeMetric(sc, "Total spilled bytes"),
+      "spilled_rows" -> SQLMetrics.createMetric(sc, "Total spilled rows"))
+  }
+
   /**
    * Creates a [[CometMetricNode]] from a [[CometPlan]].
    */
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala
index 7c335fbb..546d15c4 100644
--- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala
+++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala
@@ -933,14 +933,7 @@ case class CometSortMergeJoinExec(
     Objects.hashCode(leftKeys, rightKeys, condition, left, right)
 
   override lazy val metrics: Map[String, SQLMetric] =
-    Map(
-      "input_batches" -> SQLMetrics.createMetric(sparkContext, "Number of batches consumed"),
-      "input_rows" -> SQLMetrics.createMetric(sparkContext, "Number of rows consumed"),
-      "output_batches" -> SQLMetrics.createMetric(sparkContext, "Number of batches produced"),
-      "output_rows" -> SQLMetrics.createMetric(sparkContext, "Number of rows produced"),
-      "peak_mem_used" ->
-        SQLMetrics.createSizeMetric(sparkContext, "Peak memory used for buffered data"),
-      "join_time" -> SQLMetrics.createNanoTimingMetric(sparkContext, "Total time for joining"))
+    CometMetricNode.sortMergeJoinMetrics(sparkContext)
 }
 
 case class CometScanWrapper(override val nativeOp: Operator, override val originalPlan: SparkPlan)
diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
index bfddd74d..b632d5f8 100644
--- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala
@@ -519,6 +519,8 @@ class CometExecSuite extends CometTestBase {
           assert(metrics("peak_mem_used").value > 1L)
           assert(metrics.contains("join_time"))
           assert(metrics("join_time").value > 1L)
+          assert(metrics.contains("spill_count"))
+          assert(metrics("spill_count").value == 0)
         }
       }
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org