You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/03/22 21:49:27 UTC
(spark) branch master updated: [SPARK-55610][PYTHON] Add getExecutorInfos to StatusTracker in Python
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9518655f6241 [SPARK-55610][PYTHON] Add getExecutorInfos to StatusTracker in Python
9518655f6241 is described below
commit 9518655f6241fd47e87937c928ede4f8a39d0750
Author: Wei Liu <wl...@x.ai>
AuthorDate: Mon Mar 23 06:49:15 2026 +0900
[SPARK-55610][PYTHON] Add getExecutorInfos to StatusTracker in Python
### What changes were proposed in this pull request?
This API is not there in Python but has been there in Scala
### Why are the changes needed?
Add Python missing API
### Does this PR introduce _any_ user-facing change?
Yes, now user can access executor infos in PySpark (non-connect)
```
>>> spark.sparkContext.statusTracker().getExecutorInfos()
[SparkExecutorInfo(host=xxx, port=59602, cacheSize=0, numRunningTasks=0, usedOnHeapStorageMemory=0, usedOffHeapStorageMemory=0, totalOnHeapStorageMemory=xxx, totalOffHeapStorageMemory=0)]
```
### How was this patch tested?
Added unit, also manually tested
### Was this patch authored or co-authored using generative AI tooling?
The entire PR is written with AI except this PR description
Closes #54389 from WweiL/SPARK-55610-statustracker-executorinfos.
Authored-by: Wei Liu <wl...@x.ai>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../spark/api/java/JavaSparkStatusTracker.scala | 9 ++++-
python/pyspark/__init__.py | 3 +-
python/pyspark/core/status.py | 39 +++++++++++++++++++++-
python/pyspark/tests/test_context.py | 18 ++++++++++
4 files changed, 66 insertions(+), 3 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkStatusTracker.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkStatusTracker.scala
index 6aa290ecd7bb..0ad0224baf7a 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkStatusTracker.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkStatusTracker.scala
@@ -17,7 +17,7 @@
package org.apache.spark.api.java
-import org.apache.spark.{SparkContext, SparkJobInfo, SparkStageInfo}
+import org.apache.spark.{SparkContext, SparkExecutorInfo, SparkJobInfo, SparkStageInfo}
/**
* Low-level status reporting APIs for monitoring job and stage progress.
@@ -69,4 +69,11 @@ class JavaSparkStatusTracker private[spark] (sc: SparkContext) {
* garbage collected.
*/
def getStageInfo(stageId: Int): SparkStageInfo = sc.statusTracker.getStageInfo(stageId).orNull
+
+ /**
+ * Returns information of all known executors, including host, port, cacheSize, numRunningTasks
+ * and memory metrics.
+ * Note this include information for both the driver and executors.
+ */
+ def getExecutorInfos: Array[SparkExecutorInfo] = sc.statusTracker.getExecutorInfos
}
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index 49c594f8c7de..f2f8b08b9da7 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -55,7 +55,7 @@ from pyspark.util import is_remote_only
if not is_remote_only():
from pyspark.core.rdd import RDD, RDDBarrier
from pyspark.core.files import SparkFiles
- from pyspark.core.status import StatusTracker, SparkJobInfo, SparkStageInfo
+ from pyspark.core.status import StatusTracker, SparkJobInfo, SparkStageInfo, SparkExecutorInfo
from pyspark.core.broadcast import Broadcast
from pyspark.core import rdd, files, status, broadcast
@@ -144,6 +144,7 @@ __all__ = [
"StatusTracker",
"SparkJobInfo",
"SparkStageInfo",
+ "SparkExecutorInfo",
"Profiler",
"BasicProfiler",
"TaskContext",
diff --git a/python/pyspark/core/status.py b/python/pyspark/core/status.py
index 7e64c414403f..96eee40b2877 100644
--- a/python/pyspark/core/status.py
+++ b/python/pyspark/core/status.py
@@ -15,7 +15,7 @@
# limitations under the License.
#
-__all__ = ["SparkJobInfo", "SparkStageInfo", "StatusTracker"]
+__all__ = ["SparkJobInfo", "SparkStageInfo", "SparkExecutorInfo", "StatusTracker"]
from typing import List, NamedTuple, Optional
@@ -47,6 +47,21 @@ class SparkStageInfo(NamedTuple):
numFailedTasks: int
+class SparkExecutorInfo(NamedTuple):
+ """
+ Exposes information about Spark Executors.
+ """
+
+ host: str
+ port: int
+ cacheSize: int
+ numRunningTasks: int
+ usedOnHeapStorageMemory: int
+ usedOffHeapStorageMemory: int
+ totalOnHeapStorageMemory: int
+ totalOffHeapStorageMemory: int
+
+
class StatusTracker:
"""
Low-level status reporting APIs for monitoring job and stage progress.
@@ -110,3 +125,25 @@ class StatusTracker:
attrs = [getattr(stage, f)() for f in SparkStageInfo._fields[1:]]
return SparkStageInfo(stageId, *attrs)
return None
+
+ def getExecutorInfos(self) -> List[SparkExecutorInfo]:
+ """
+ Returns a list of :class:`SparkExecutorInfo`,
+ contains information of all known executors, including host, port, cacheSize,
+ numRunningTasks and memory metrics.
+ Note this includes information for both the driver and executors.
+ """
+ executor_infos = self._jtracker.getExecutorInfos()
+ return [
+ SparkExecutorInfo(
+ exec_info.host(),
+ exec_info.port(),
+ exec_info.cacheSize(),
+ exec_info.numRunningTasks(),
+ exec_info.usedOnHeapStorageMemory(),
+ exec_info.usedOffHeapStorageMemory(),
+ exec_info.totalOnHeapStorageMemory(),
+ exec_info.totalOffHeapStorageMemory(),
+ )
+ for exec_info in executor_infos
+ ]
diff --git a/python/pyspark/tests/test_context.py b/python/pyspark/tests/test_context.py
index 13dcd363bc79..66bb0ebdd097 100644
--- a/python/pyspark/tests/test_context.py
+++ b/python/pyspark/tests/test_context.py
@@ -263,6 +263,24 @@ class ContextTests(unittest.TestCase):
sc.stop()
+ def test_executor_infos(self):
+ with SparkContext() as sc:
+ tracker = sc.statusTracker()
+ executorInfos = tracker.getExecutorInfos()
+ # In local mode there should be at least one executor (the driver)
+ self.assertGreaterEqual(len(executorInfos), 1)
+ for info in executorInfos:
+ self.assertIsNotNone(info.host)
+ self.assertGreaterEqual(info.port, 0)
+ self.assertGreaterEqual(info.cacheSize, 0)
+ self.assertGreaterEqual(info.numRunningTasks, 0)
+ self.assertGreaterEqual(info.usedOnHeapStorageMemory, 0)
+ self.assertGreaterEqual(info.usedOffHeapStorageMemory, 0)
+ self.assertGreaterEqual(info.totalOnHeapStorageMemory, 0)
+ self.assertGreaterEqual(info.totalOffHeapStorageMemory, 0)
+
+ sc.stop()
+
def test_startTime(self):
with SparkContext() as sc:
self.assertGreater(sc.startTime, 0)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org