You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/03/22 21:49:27 UTC

(spark) branch master updated: [SPARK-55610][PYTHON] Add getExecutorInfos to StatusTracker in Python

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 9518655f6241 [SPARK-55610][PYTHON] Add getExecutorInfos to StatusTracker in Python
9518655f6241 is described below

commit 9518655f6241fd47e87937c928ede4f8a39d0750
Author: Wei Liu <wl...@x.ai>
AuthorDate: Mon Mar 23 06:49:15 2026 +0900

    [SPARK-55610][PYTHON] Add getExecutorInfos to StatusTracker in Python
    
    ### What changes were proposed in this pull request?
    
    This API is not there in Python but has been there in Scala
    
    ### Why are the changes needed?
    
    Add Python missing API
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, now user can access executor infos in PySpark (non-connect)
    ```
    >>> spark.sparkContext.statusTracker().getExecutorInfos()
    [SparkExecutorInfo(host=xxx, port=59602, cacheSize=0, numRunningTasks=0, usedOnHeapStorageMemory=0, usedOffHeapStorageMemory=0, totalOnHeapStorageMemory=xxx, totalOffHeapStorageMemory=0)]
    ```
    
    ### How was this patch tested?
    
    Added unit, also manually tested
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    The entire PR is written with AI except this PR description
    
    Closes #54389 from WweiL/SPARK-55610-statustracker-executorinfos.
    
    Authored-by: Wei Liu <wl...@x.ai>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 .../spark/api/java/JavaSparkStatusTracker.scala    |  9 ++++-
 python/pyspark/__init__.py                         |  3 +-
 python/pyspark/core/status.py                      | 39 +++++++++++++++++++++-
 python/pyspark/tests/test_context.py               | 18 ++++++++++
 4 files changed, 66 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkStatusTracker.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkStatusTracker.scala
index 6aa290ecd7bb..0ad0224baf7a 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkStatusTracker.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkStatusTracker.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.api.java
 
-import org.apache.spark.{SparkContext, SparkJobInfo, SparkStageInfo}
+import org.apache.spark.{SparkContext, SparkExecutorInfo, SparkJobInfo, SparkStageInfo}
 
 /**
  * Low-level status reporting APIs for monitoring job and stage progress.
@@ -69,4 +69,11 @@ class JavaSparkStatusTracker private[spark] (sc: SparkContext) {
    * garbage collected.
    */
   def getStageInfo(stageId: Int): SparkStageInfo = sc.statusTracker.getStageInfo(stageId).orNull
+
+  /**
+   * Returns information of all known executors, including host, port, cacheSize, numRunningTasks
+   * and memory metrics.
+   * Note this include information for both the driver and executors.
+   */
+  def getExecutorInfos: Array[SparkExecutorInfo] = sc.statusTracker.getExecutorInfos
 }
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index 49c594f8c7de..f2f8b08b9da7 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -55,7 +55,7 @@ from pyspark.util import is_remote_only
 if not is_remote_only():
     from pyspark.core.rdd import RDD, RDDBarrier
     from pyspark.core.files import SparkFiles
-    from pyspark.core.status import StatusTracker, SparkJobInfo, SparkStageInfo
+    from pyspark.core.status import StatusTracker, SparkJobInfo, SparkStageInfo, SparkExecutorInfo
     from pyspark.core.broadcast import Broadcast
     from pyspark.core import rdd, files, status, broadcast
 
@@ -144,6 +144,7 @@ __all__ = [
     "StatusTracker",
     "SparkJobInfo",
     "SparkStageInfo",
+    "SparkExecutorInfo",
     "Profiler",
     "BasicProfiler",
     "TaskContext",
diff --git a/python/pyspark/core/status.py b/python/pyspark/core/status.py
index 7e64c414403f..96eee40b2877 100644
--- a/python/pyspark/core/status.py
+++ b/python/pyspark/core/status.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-__all__ = ["SparkJobInfo", "SparkStageInfo", "StatusTracker"]
+__all__ = ["SparkJobInfo", "SparkStageInfo", "SparkExecutorInfo", "StatusTracker"]
 
 from typing import List, NamedTuple, Optional
 
@@ -47,6 +47,21 @@ class SparkStageInfo(NamedTuple):
     numFailedTasks: int
 
 
+class SparkExecutorInfo(NamedTuple):
+    """
+    Exposes information about Spark Executors.
+    """
+
+    host: str
+    port: int
+    cacheSize: int
+    numRunningTasks: int
+    usedOnHeapStorageMemory: int
+    usedOffHeapStorageMemory: int
+    totalOnHeapStorageMemory: int
+    totalOffHeapStorageMemory: int
+
+
 class StatusTracker:
     """
     Low-level status reporting APIs for monitoring job and stage progress.
@@ -110,3 +125,25 @@ class StatusTracker:
             attrs = [getattr(stage, f)() for f in SparkStageInfo._fields[1:]]
             return SparkStageInfo(stageId, *attrs)
         return None
+
+    def getExecutorInfos(self) -> List[SparkExecutorInfo]:
+        """
+        Returns a list of :class:`SparkExecutorInfo`,
+        contains information of all known executors, including host, port, cacheSize,
+        numRunningTasks and memory metrics.
+        Note this includes information for both the driver and executors.
+        """
+        executor_infos = self._jtracker.getExecutorInfos()
+        return [
+            SparkExecutorInfo(
+                exec_info.host(),
+                exec_info.port(),
+                exec_info.cacheSize(),
+                exec_info.numRunningTasks(),
+                exec_info.usedOnHeapStorageMemory(),
+                exec_info.usedOffHeapStorageMemory(),
+                exec_info.totalOnHeapStorageMemory(),
+                exec_info.totalOffHeapStorageMemory(),
+            )
+            for exec_info in executor_infos
+        ]
diff --git a/python/pyspark/tests/test_context.py b/python/pyspark/tests/test_context.py
index 13dcd363bc79..66bb0ebdd097 100644
--- a/python/pyspark/tests/test_context.py
+++ b/python/pyspark/tests/test_context.py
@@ -263,6 +263,24 @@ class ContextTests(unittest.TestCase):
 
             sc.stop()
 
+    def test_executor_infos(self):
+        with SparkContext() as sc:
+            tracker = sc.statusTracker()
+            executorInfos = tracker.getExecutorInfos()
+            # In local mode there should be at least one executor (the driver)
+            self.assertGreaterEqual(len(executorInfos), 1)
+            for info in executorInfos:
+                self.assertIsNotNone(info.host)
+                self.assertGreaterEqual(info.port, 0)
+                self.assertGreaterEqual(info.cacheSize, 0)
+                self.assertGreaterEqual(info.numRunningTasks, 0)
+                self.assertGreaterEqual(info.usedOnHeapStorageMemory, 0)
+                self.assertGreaterEqual(info.usedOffHeapStorageMemory, 0)
+                self.assertGreaterEqual(info.totalOnHeapStorageMemory, 0)
+                self.assertGreaterEqual(info.totalOffHeapStorageMemory, 0)
+
+            sc.stop()
+
     def test_startTime(self):
         with SparkContext() as sc:
             self.assertGreater(sc.startTime, 0)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org