You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2022/03/31 13:43:33 UTC
[spark] branch master updated: [SPARK-37831][CORE] add task partition id in TaskInfo and Task Metrics
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 13cfc5b [SPARK-37831][CORE] add task partition id in TaskInfo and Task Metrics
13cfc5b is described below
commit 13cfc5bcaf8ebda8769595c58902327c4affdecf
Author: stczwd <qc...@163.com>
AuthorDate: Thu Mar 31 21:42:23 2022 +0800
[SPARK-37831][CORE] add task partition id in TaskInfo and Task Metrics
### Why are the changes needed?
There is no partition id in the current task metrics. It is difficult to track the task metrics of a specific partition or the stage metrics of processing data, especially when the stage was retried.
```
class TaskData private[spark](
val taskId: Long,
val index: Int,
val attempt: Int,
val launchTime: Date,
val resultFetchStart: Option[Date],
JsonDeserialize(contentAs = classOf[JLong])
val duration: Option[Long],
val executorId: String,
val host: String,
val status: String,
val taskLocality: String,
val speculative: Boolean,
val accumulatorUpdates: Seq[AccumulableInfo],
val errorMessage: Option[String] = None,
val taskMetrics: Option[TaskMetrics] = None,
val executorLogs: Map[String, String],
val schedulerDelay: Long,
val gettingResultTime: Long)
```
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
add new tests
Closes #35185 from jackylee-ch/SPARK-37831.
Lead-authored-by: stczwd <qc...@163.com>
Co-authored-by: Jacky Lee <qc...@gmail.com>
Co-authored-by: jackylee-ch <li...@baidu.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../org/apache/spark/scheduler/TaskInfo.scala | 22 +
.../apache/spark/scheduler/TaskSetManager.scala | 3 +-
.../org/apache/spark/status/AppStatusStore.scala | 3 +-
.../scala/org/apache/spark/status/LiveEntity.scala | 1 +
.../scala/org/apache/spark/status/api/v1/api.scala | 1 +
.../scala/org/apache/spark/status/storeTypes.scala | 6 +
.../scala/org/apache/spark/util/JsonProtocol.scala | 7 +-
.../application_list_json_expectation.json | 15 +
.../completed_app_list_json_expectation.json | 15 +
.../excludeOnFailure_for_stage_expectation.json | 12 +
...xcludeOnFailure_node_for_stage_expectation.json | 14 +
.../limit_app_list_json_expectation.json | 34 +-
.../minDate_app_list_json_expectation.json | 15 +
.../minEndDate_app_list_json_expectation.json | 15 +
.../one_stage_attempt_json_expectation.json | 8 +
.../one_stage_json_expectation.json | 8 +
.../one_stage_json_with_details_expectation.json | 8 +
...ne_stage_json_with_partitionId_expectation.json | 624 +++++++++++++++++++++
.../stage_task_list_expectation.json | 21 +
...from_multi_attempt_app_json_1__expectation.json | 8 +
...from_multi_attempt_app_json_2__expectation.json | 8 +
...e_task_list_w__offset___length_expectation.json | 50 ++
.../stage_task_list_w__sortBy_expectation.json | 20 +
...__sortBy_short_names___runtime_expectation.json | 20 +
...w__sortBy_short_names__runtime_expectation.json | 20 +
...st_w__status___offset___length_expectation.json | 2 +
...___sortBy_short_names__runtime_expectation.json | 20 +
.../stage_task_list_w__status_expectation.json | 10 +
...ge_task_list_with_partitionId_expectation.json} | 302 +++++-----
.../stage_with_accumulable_json_expectation.json | 8 +
.../stage_with_peak_metrics_expectation.json | 16 +
...stage_with_speculation_summary_expectation.json | 5 +
.../stage_with_summaries_expectation.json | 16 +
.../resources/spark-events/local-1642039451826 | 115 ++++
.../spark/ExecutorAllocationManagerSuite.scala | 7 +-
.../spark/deploy/history/HistoryServerSuite.scala | 2 +
.../apache/spark/scheduler/DAGSchedulerSuite.scala | 7 +-
.../scheduler/EventLoggingListenerSuite.scala | 3 +-
.../scheduler/dynalloc/ExecutorMonitorSuite.scala | 3 +-
.../spark/status/AppStatusListenerSuite.scala | 7 +-
.../apache/spark/status/AppStatusStoreSuite.scala | 8 +-
.../apache/spark/status/AppStatusUtilsSuite.scala | 2 +
.../spark/status/ListenerEventsTestHelper.scala | 5 +-
.../scala/org/apache/spark/ui/StagePageSuite.scala | 4 +-
.../org/apache/spark/util/JsonProtocolSuite.scala | 23 +-
dev/.rat-excludes | 1 +
project/MimaExcludes.scala | 3 +
.../execution/ui/MetricsAggregationBenchmark.scala | 1 +
.../execution/ui/SQLAppStatusListenerSuite.scala | 1 +
49 files changed, 1315 insertions(+), 214 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala
index 67799b2..b135a82 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala
@@ -35,6 +35,12 @@ class TaskInfo(
*/
val index: Int,
val attemptNumber: Int,
+ /**
+ * The actual RDD partition ID in this task.
+ * The ID of the RDD partition is always same across task attempts.
+ * This will be -1 for historical data, and available for all applications since Spark 3.3.
+ */
+ val partitionId: Int,
val launchTime: Long,
val executorId: String,
val host: String,
@@ -42,6 +48,22 @@ class TaskInfo(
val speculative: Boolean) {
/**
+ * This api doesn't contains partitionId, please use the new api.
+ * Remain it for backward compatibility before Spark 3.3.
+ */
+ def this(
+ taskId: Long,
+ index: Int,
+ attemptNumber: Int,
+ launchTime: Long,
+ executorId: String,
+ host: String,
+ taskLocality: TaskLocality.TaskLocality,
+ speculative: Boolean) = {
+ this(taskId, index, attemptNumber, -1, launchTime, executorId, host, taskLocality, speculative)
+ }
+
+ /**
* The time when the task started remotely getting the result. Will not be set if the
* task result was sent immediately when the task finished (as opposed to sending an
* IndirectTaskResult and later fetching the result from the block manager).
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index b7fae2a..8523dc4 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -509,7 +509,8 @@ private[spark] class TaskSetManager(
// Do various bookkeeping
copiesRunning(index) += 1
val attemptNum = taskAttempts(index).size
- val info = new TaskInfo(taskId, index, attemptNum, launchTime,
+ val info = new TaskInfo(
+ taskId, index, attemptNum, task.partitionId, launchTime,
execId, host, taskLocality, speculative)
taskInfos(taskId) = info
taskAttempts(index) = info :: taskAttempts(index)
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 398cd45..03767ee 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -743,7 +743,8 @@ private[spark] class AppStatusStore(
})
new v1.TaskData(taskDataOld.taskId, taskDataOld.index,
- taskDataOld.attempt, taskDataOld.launchTime, taskDataOld.resultFetchStart,
+ taskDataOld.attempt, taskDataOld.partitionId,
+ taskDataOld.launchTime, taskDataOld.resultFetchStart,
taskDataOld.duration, taskDataOld.executorId, taskDataOld.host, taskDataOld.status,
taskDataOld.taskLocality, taskDataOld.speculative, taskDataOld.accumulatorUpdates,
taskDataOld.errorMessage, taskDataOld.taskMetrics,
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index c5d72d3..6437d2f 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -201,6 +201,7 @@ private class LiveTask(
info.taskId,
info.index,
info.attemptNumber,
+ info.partitionId,
info.launchTime,
if (info.gettingResult) info.gettingResultTime else -1L,
duration,
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 86ddd3b..d1a4d60 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -308,6 +308,7 @@ class TaskData private[spark](
val taskId: Long,
val index: Int,
val attempt: Int,
+ val partitionId: Int,
val launchTime: Date,
val resultFetchStart: Option[Date],
@JsonDeserialize(contentAs = classOf[JLong])
diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
index 39bf593..895fb58 100644
--- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala
+++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
@@ -141,6 +141,7 @@ private[spark] object TaskIndexNames {
final val STAGE = "stage"
final val STATUS = "sta"
final val TASK_INDEX = "idx"
+ final val TASK_PARTITION_ID = "partid"
final val COMPLETION_TIME = "ct"
}
@@ -161,6 +162,10 @@ private[spark] class TaskDataWrapper(
val index: Int,
@KVIndexParam(value = TaskIndexNames.ATTEMPT, parent = TaskIndexNames.STAGE)
val attempt: Int,
+ @KVIndexParam(value = TaskIndexNames.TASK_PARTITION_ID, parent = TaskIndexNames.STAGE)
+ // Different kvstores have different default values (eg 0 or -1),
+ // thus we use the default value here for backwards compatibility.
+ val partitionId: Int = -1,
@KVIndexParam(value = TaskIndexNames.LAUNCH_TIME, parent = TaskIndexNames.STAGE)
val launchTime: Long,
val resultFetchStart: Long,
@@ -286,6 +291,7 @@ private[spark] class TaskDataWrapper(
taskId,
index,
attempt,
+ partitionId,
new Date(launchTime),
if (resultFetchStart > 0L) Some(new Date(resultFetchStart)) else None,
if (duration > 0L) Some(duration) else None,
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index f9b6ed3..acbd323 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -317,6 +317,7 @@ private[spark] object JsonProtocol {
("Task ID" -> taskInfo.taskId) ~
("Index" -> taskInfo.index) ~
("Attempt" -> taskInfo.attemptNumber) ~
+ ("Partition ID" -> taskInfo.partitionId) ~
("Launch Time" -> taskInfo.launchTime) ~
("Executor ID" -> taskInfo.executorId) ~
("Host" -> taskInfo.host) ~
@@ -916,6 +917,7 @@ private[spark] object JsonProtocol {
val taskId = (json \ "Task ID").extract[Long]
val index = (json \ "Index").extract[Int]
val attempt = jsonOption(json \ "Attempt").map(_.extract[Int]).getOrElse(1)
+ val partitionId = jsonOption(json \ "Partition ID").map(_.extract[Int]).getOrElse(-1)
val launchTime = (json \ "Launch Time").extract[Long]
val executorId = weakIntern((json \ "Executor ID").extract[String])
val host = weakIntern((json \ "Host").extract[String])
@@ -930,8 +932,9 @@ private[spark] object JsonProtocol {
case None => Seq.empty[AccumulableInfo]
}
- val taskInfo =
- new TaskInfo(taskId, index, attempt, launchTime, executorId, host, taskLocality, speculative)
+ val taskInfo = new TaskInfo(
+ taskId, index, attempt, partitionId, launchTime,
+ executorId, host, taskLocality, speculative)
taskInfo.gettingResultTime = gettingResultTime
taskInfo.finishTime = finishTime
taskInfo.failed = failed
diff --git a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
index ae1edbc..e15707c 100644
--- a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
@@ -1,4 +1,19 @@
[ {
+ "id" : "local-1642039451826",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2022-01-13T02:04:10.519GMT",
+ "endTime" : "2022-01-13T02:05:36.564GMT",
+ "lastUpdated" : "",
+ "duration" : 86045,
+ "sparkUser" : "lijunqing",
+ "completed" : true,
+ "appSparkVersion" : "3.3.0-SNAPSHOT",
+ "startTimeEpoch" : 1642039450519,
+ "endTimeEpoch" : 1642039536564,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
"id" : "application_1628109047826_1317105",
"name" : "Spark shell",
"attempts" : [ {
diff --git a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
index ae1edbc..e15707c 100644
--- a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
@@ -1,4 +1,19 @@
[ {
+ "id" : "local-1642039451826",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2022-01-13T02:04:10.519GMT",
+ "endTime" : "2022-01-13T02:05:36.564GMT",
+ "lastUpdated" : "",
+ "duration" : 86045,
+ "sparkUser" : "lijunqing",
+ "completed" : true,
+ "appSparkVersion" : "3.3.0-SNAPSHOT",
+ "startTimeEpoch" : 1642039450519,
+ "endTimeEpoch" : 1642039536564,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
"id" : "application_1628109047826_1317105",
"name" : "Spark shell",
"attempts" : [ {
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
index 9be6b6e..69747c8 100644
--- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
@@ -46,6 +46,7 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:18.347GMT",
"duration" : 562,
"executorId" : "0",
@@ -100,6 +101,7 @@
"taskId" : 5,
"index" : 3,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:18.958GMT",
"duration" : 22,
"executorId" : "1",
@@ -153,6 +155,7 @@
"taskId" : 10,
"index" : 8,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:19.034GMT",
"duration" : 12,
"executorId" : "1",
@@ -206,6 +209,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:18.364GMT",
"duration" : 565,
"executorId" : "1",
@@ -259,6 +263,7 @@
"taskId" : 6,
"index" : 4,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:18.980GMT",
"duration" : 16,
"executorId" : "1",
@@ -312,6 +317,7 @@
"taskId" : 9,
"index" : 7,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:19.022GMT",
"duration" : 12,
"executorId" : "1",
@@ -365,6 +371,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:18.899GMT",
"duration" : 27,
"executorId" : "0",
@@ -419,6 +426,7 @@
"taskId" : 7,
"index" : 5,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:18.996GMT",
"duration" : 15,
"executorId" : "1",
@@ -472,6 +480,7 @@
"taskId" : 3,
"index" : 0,
"attempt" : 1,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:18.919GMT",
"duration" : 24,
"executorId" : "1",
@@ -525,6 +534,7 @@
"taskId" : 11,
"index" : 9,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:19.045GMT",
"duration" : 15,
"executorId" : "1",
@@ -578,6 +588,7 @@
"taskId" : 8,
"index" : 6,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:19.011GMT",
"duration" : 11,
"executorId" : "1",
@@ -631,6 +642,7 @@
"taskId" : 4,
"index" : 2,
"attempt" : 1,
+ "partitionId": -1,
"launchTime" : "2018-01-09T10:21:18.943GMT",
"duration" : 16,
"executorId" : "1",
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
index 1661f7c..35bee44 100644
--- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
@@ -46,6 +46,7 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:12.816GMT",
"duration" : 2064,
"executorId" : "1",
@@ -99,6 +100,7 @@
"taskId" : 5,
"index" : 5,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:14.320GMT",
"duration" : 73,
"executorId" : "5",
@@ -153,6 +155,7 @@
"taskId" : 10,
"index" : 1,
"attempt" : 1,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:15.069GMT",
"duration" : 132,
"executorId" : "2",
@@ -206,6 +209,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:12.832GMT",
"duration" : 1506,
"executorId" : "5",
@@ -260,6 +264,7 @@
"taskId" : 6,
"index" : 6,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:14.323GMT",
"duration" : 67,
"executorId" : "4",
@@ -314,6 +319,7 @@
"taskId" : 9,
"index" : 4,
"attempt" : 1,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:14.973GMT",
"duration" : 96,
"executorId" : "2",
@@ -367,6 +373,7 @@
"taskId" : 13,
"index" : 9,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:15.200GMT",
"duration" : 76,
"executorId" : "2",
@@ -420,6 +427,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:12.832GMT",
"duration" : 1774,
"executorId" : "3",
@@ -473,6 +481,7 @@
"taskId" : 12,
"index" : 8,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:15.165GMT",
"duration" : 60,
"executorId" : "1",
@@ -526,6 +535,7 @@
"taskId" : 7,
"index" : 5,
"attempt" : 1,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:14.859GMT",
"duration" : 115,
"executorId" : "2",
@@ -579,6 +589,7 @@
"taskId" : 3,
"index" : 3,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:12.833GMT",
"duration" : 2027,
"executorId" : "2",
@@ -632,6 +643,7 @@
"taskId" : 11,
"index" : 7,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:15.072GMT",
"duration" : 93,
"executorId" : "1",
@@ -685,6 +697,7 @@
"taskId" : 8,
"index" : 6,
"attempt" : 1,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:14.879GMT",
"duration" : 194,
"executorId" : "1",
@@ -738,6 +751,7 @@
"taskId" : 4,
"index" : 4,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2018-01-18T18:33:12.833GMT",
"duration" : 1522,
"executorId" : "4",
diff --git a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
index 91e3ebd..2a504ce 100644
--- a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
@@ -1,4 +1,19 @@
[ {
+ "id" : "local-1642039451826",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2022-01-13T02:04:10.519GMT",
+ "endTime" : "2022-01-13T02:05:36.564GMT",
+ "lastUpdated" : "",
+ "duration" : 86045,
+ "sparkUser" : "lijunqing",
+ "completed" : true,
+ "appSparkVersion" : "3.3.0-SNAPSHOT",
+ "startTimeEpoch" : 1642039450519,
+ "endTimeEpoch" : 1642039536564,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
"id" : "application_1628109047826_1317105",
"name" : "Spark shell",
"attempts" : [ {
@@ -24,23 +39,8 @@
"sparkUser" : "terryk",
"completed" : true,
"appSparkVersion" : "3.1.0-SNAPSHOT",
- "lastUpdatedEpoch" : 0,
+ "startTimeEpoch" : 1594091460235,
"endTimeEpoch" : 1594091824231,
- "startTimeEpoch" : 1594091460235
- } ]
-}, {
- "id" : "application_1578436911597_0052",
- "name" : "Spark shell",
- "attempts" : [ {
- "startTime" : "2020-01-11T17:44:22.851GMT",
- "endTime" : "2020-01-11T17:46:42.615GMT",
- "lastUpdated" : "",
- "duration" : 139764,
- "sparkUser" : "tgraves",
- "completed" : true,
- "appSparkVersion" : "3.0.0-SNAPSHOT",
- "endTimeEpoch" : 1578764802615,
- "startTimeEpoch" : 1578764662851,
"lastUpdatedEpoch" : 0
} ]
-} ]
+} ]
\ No newline at end of file
diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
index 9885d36..3892063 100644
--- a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
@@ -1,4 +1,19 @@
[ {
+ "id" : "local-1642039451826",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2022-01-13T02:04:10.519GMT",
+ "endTime" : "2022-01-13T02:05:36.564GMT",
+ "lastUpdated" : "",
+ "duration" : 86045,
+ "sparkUser" : "lijunqing",
+ "completed" : true,
+ "appSparkVersion" : "3.3.0-SNAPSHOT",
+ "startTimeEpoch" : 1642039450519,
+ "endTimeEpoch" : 1642039536564,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
"id" : "application_1628109047826_1317105",
"name" : "Spark shell",
"attempts" : [ {
diff --git a/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
index cf0bad7..3bd637a 100644
--- a/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
@@ -1,4 +1,19 @@
[ {
+ "id" : "local-1642039451826",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2022-01-13T02:04:10.519GMT",
+ "endTime" : "2022-01-13T02:05:36.564GMT",
+ "lastUpdated" : "",
+ "duration" : 86045,
+ "sparkUser" : "lijunqing",
+ "completed" : true,
+ "appSparkVersion" : "3.3.0-SNAPSHOT",
+ "startTimeEpoch" : 1642039450519,
+ "endTimeEpoch" : 1642039536564,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
"id" : "application_1628109047826_1317105",
"name" : "Spark shell",
"attempts" : [ {
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index 94d343d..9b9da9a 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -46,6 +46,7 @@
"taskId" : 10,
"index" : 2,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"duration" : 456,
"executorId" : "driver",
@@ -96,6 +97,7 @@
"taskId" : 14,
"index" : 6,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.832GMT",
"duration" : 450,
"executorId" : "driver",
@@ -146,6 +148,7 @@
"taskId" : 9,
"index" : 1,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"duration" : 454,
"executorId" : "driver",
@@ -196,6 +199,7 @@
"taskId" : 13,
"index" : 5,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.831GMT",
"duration" : 452,
"executorId" : "driver",
@@ -246,6 +250,7 @@
"taskId" : 12,
"index" : 4,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.831GMT",
"duration" : 454,
"executorId" : "driver",
@@ -296,6 +301,7 @@
"taskId" : 11,
"index" : 3,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"duration" : 454,
"executorId" : "driver",
@@ -346,6 +352,7 @@
"taskId" : 8,
"index" : 0,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.829GMT",
"duration" : 454,
"executorId" : "driver",
@@ -396,6 +403,7 @@
"taskId" : 15,
"index" : 7,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.833GMT",
"duration" : 450,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 6713486..f57feca 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -46,6 +46,7 @@
"taskId" : 10,
"index" : 2,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"duration" : 456,
"executorId" : "driver",
@@ -96,6 +97,7 @@
"taskId" : 14,
"index" : 6,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.832GMT",
"duration" : 450,
"executorId" : "driver",
@@ -146,6 +148,7 @@
"taskId" : 9,
"index" : 1,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"duration" : 454,
"executorId" : "driver",
@@ -196,6 +199,7 @@
"taskId" : 13,
"index" : 5,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.831GMT",
"duration" : 452,
"executorId" : "driver",
@@ -246,6 +250,7 @@
"taskId" : 12,
"index" : 4,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.831GMT",
"duration" : 454,
"executorId" : "driver",
@@ -296,6 +301,7 @@
"taskId" : 11,
"index" : 3,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"duration" : 454,
"executorId" : "driver",
@@ -346,6 +352,7 @@
"taskId" : 8,
"index" : 0,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.829GMT",
"duration" : 454,
"executorId" : "driver",
@@ -396,6 +403,7 @@
"taskId" : 15,
"index" : 7,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.833GMT",
"duration" : 450,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
index 6713486..f57feca 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
@@ -46,6 +46,7 @@
"taskId" : 10,
"index" : 2,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"duration" : 456,
"executorId" : "driver",
@@ -96,6 +97,7 @@
"taskId" : 14,
"index" : 6,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.832GMT",
"duration" : 450,
"executorId" : "driver",
@@ -146,6 +148,7 @@
"taskId" : 9,
"index" : 1,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"duration" : 454,
"executorId" : "driver",
@@ -196,6 +199,7 @@
"taskId" : 13,
"index" : 5,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.831GMT",
"duration" : 452,
"executorId" : "driver",
@@ -246,6 +250,7 @@
"taskId" : 12,
"index" : 4,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.831GMT",
"duration" : 454,
"executorId" : "driver",
@@ -296,6 +301,7 @@
"taskId" : 11,
"index" : 3,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"duration" : 454,
"executorId" : "driver",
@@ -346,6 +352,7 @@
"taskId" : 8,
"index" : 0,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.829GMT",
"duration" : 454,
"executorId" : "driver",
@@ -396,6 +403,7 @@
"taskId" : 15,
"index" : 7,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-02-03T16:43:05.833GMT",
"duration" : 450,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
new file mode 100644
index 0000000..cc944e7
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
@@ -0,0 +1,624 @@
+[ {
+ "status" : "COMPLETE",
+ "stageId" : 2,
+ "attemptId" : 0,
+ "numTasks" : 10,
+ "numActiveTasks" : 0,
+ "numCompleteTasks" : 10,
+ "numFailedTasks" : 0,
+ "numKilledTasks" : 0,
+ "numCompletedIndices" : 10,
+ "submissionTime" : "2022-01-13T02:04:57.017GMT",
+ "firstTaskLaunchedTime" : "2022-01-13T02:04:57.053GMT",
+ "completionTime" : "2022-01-13T02:04:57.127GMT",
+ "executorDeserializeTime" : 33,
+ "executorDeserializeCpuTime" : 25578000,
+ "executorRunTime" : 455,
+ "executorCpuTime" : 173422000,
+ "resultSize" : 28480,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputBytes" : 0,
+ "inputRecords" : 0,
+ "outputBytes" : 0,
+ "outputRecords" : 0,
+ "shuffleRemoteBlocksFetched" : 0,
+ "shuffleLocalBlocksFetched" : 80,
+ "shuffleFetchWaitTime" : 0,
+ "shuffleRemoteBytesRead" : 0,
+ "shuffleRemoteBytesReadToDisk" : 0,
+ "shuffleLocalBytesRead" : 3760,
+ "shuffleReadBytes" : 3760,
+ "shuffleReadRecords" : 100,
+ "shuffleWriteBytes" : 590,
+ "shuffleWriteTime" : 10569751,
+ "shuffleWriteRecords" : 10,
+ "name" : "count at <console>:23",
+ "details" : "org.apache.spark.sql.Dataset.count(Dataset.scala:3130)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:23)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:27)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:29)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:31)\n$line15.$read$$iw$$iw$$iw$$iw.<init>(<console>:33)\n$line15.$read$$iw$$iw$$iw.<init>(<console>:35)\n$line15.$read$$iw$$iw.<init>(<console>:37)\n$line15.$read$$iw.<init>(<con [...]
+ "schedulingPool" : "default",
+ "rddIds" : [ 7, 6, 5 ],
+ "accumulatorUpdates" : [ ],
+ "tasks" : {
+ "10" : {
+ "taskId" : 10,
+ "index" : 2,
+ "attempt" : 0,
+ "partitionId" : 2,
+ "launchTime" : "2022-01-13T02:04:57.055GMT",
+ "duration" : 61,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 4,
+ "executorDeserializeCpuTime" : 2238000,
+ "executorRunTime" : 52,
+ "executorCpuTime" : 27082000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 376,
+ "recordsRead" : 10
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 1282083,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
+ },
+ "14" : {
+ "taskId" : 14,
+ "index" : 6,
+ "attempt" : 0,
+ "partitionId" : 6,
+ "launchTime" : "2022-01-13T02:04:57.056GMT",
+ "duration" : 59,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 4,
+ "executorDeserializeCpuTime" : 2498000,
+ "executorRunTime" : 50,
+ "executorCpuTime" : 21709000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 376,
+ "recordsRead" : 10
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 976166,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 5,
+ "gettingResultTime" : 0
+ },
+ "9" : {
+ "taskId" : 9,
+ "index" : 1,
+ "attempt" : 0,
+ "partitionId" : 1,
+ "launchTime" : "2022-01-13T02:04:57.055GMT",
+ "duration" : 65,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 4,
+ "executorDeserializeCpuTime" : 3175000,
+ "executorRunTime" : 57,
+ "executorCpuTime" : 18115000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 372,
+ "recordsRead" : 9
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 1490249,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
+ },
+ "13" : {
+ "taskId" : 13,
+ "index" : 5,
+ "attempt" : 0,
+ "partitionId" : 5,
+ "launchTime" : "2022-01-13T02:04:57.056GMT",
+ "duration" : 67,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 3,
+ "executorDeserializeCpuTime" : 2177000,
+ "executorRunTime" : 60,
+ "executorCpuTime" : 18621000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 372,
+ "recordsRead" : 9
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 673209,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
+ },
+ "17" : {
+ "taskId" : 17,
+ "index" : 9,
+ "attempt" : 0,
+ "partitionId" : 9,
+ "launchTime" : "2022-01-13T02:04:57.115GMT",
+ "duration" : 10,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 1,
+ "executorDeserializeCpuTime" : 1662000,
+ "executorRunTime" : 6,
+ "executorCpuTime" : 5470000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 380,
+ "recordsRead" : 11
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 506707,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 3,
+ "gettingResultTime" : 0
+ },
+ "12" : {
+ "taskId" : 12,
+ "index" : 4,
+ "attempt" : 0,
+ "partitionId" : 4,
+ "launchTime" : "2022-01-13T02:04:57.056GMT",
+ "duration" : 62,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 4,
+ "executorDeserializeCpuTime" : 2614000,
+ "executorRunTime" : 54,
+ "executorCpuTime" : 21689000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 372,
+ "recordsRead" : 9
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 1093250,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
+ },
+ "16" : {
+ "taskId" : 16,
+ "index" : 8,
+ "attempt" : 0,
+ "partitionId" : 8,
+ "launchTime" : "2022-01-13T02:04:57.114GMT",
+ "duration" : 12,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 1,
+ "executorDeserializeCpuTime" : 1875000,
+ "executorRunTime" : 7,
+ "executorCpuTime" : 5421000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 380,
+ "recordsRead" : 11
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 568086,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
+ },
+ "11" : {
+ "taskId" : 11,
+ "index" : 3,
+ "attempt" : 0,
+ "partitionId" : 3,
+ "launchTime" : "2022-01-13T02:04:57.055GMT",
+ "duration" : 64,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 6,
+ "executorDeserializeCpuTime" : 3452000,
+ "executorRunTime" : 54,
+ "executorCpuTime" : 17668000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 376,
+ "recordsRead" : 10
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 1558374,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 4,
+ "gettingResultTime" : 0
+ },
+ "8" : {
+ "taskId" : 8,
+ "index" : 0,
+ "attempt" : 0,
+ "partitionId" : 0,
+ "launchTime" : "2022-01-13T02:04:57.053GMT",
+ "duration" : 71,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 3,
+ "executorDeserializeCpuTime" : 3128000,
+ "executorRunTime" : 62,
+ "executorCpuTime" : 20132000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 376,
+ "recordsRead" : 10
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 1340668,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
+ },
+ "15" : {
+ "taskId" : 15,
+ "index" : 7,
+ "attempt" : 0,
+ "partitionId" : 7,
+ "launchTime" : "2022-01-13T02:04:57.056GMT",
+ "duration" : 62,
+ "executorId" : "driver",
+ "host" : "172.22.200.52",
+ "status" : "SUCCESS",
+ "taskLocality" : "NODE_LOCAL",
+ "speculative" : false,
+ "accumulatorUpdates" : [ ],
+ "taskMetrics" : {
+ "executorDeserializeTime" : 3,
+ "executorDeserializeCpuTime" : 2759000,
+ "executorRunTime" : 53,
+ "executorCpuTime" : 17515000,
+ "resultSize" : 2848,
+ "jvmGcTime" : 0,
+ "resultSerializationTime" : 0,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "peakExecutionMemory" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 8,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "remoteBytesReadToDisk" : 0,
+ "localBytesRead" : 380,
+ "recordsRead" : 11
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 59,
+ "writeTime" : 1080959,
+ "recordsWritten" : 1
+ }
+ },
+ "executorLogs" : { },
+ "schedulerDelay" : 6,
+ "gettingResultTime" : 0
+ }
+ },
+ "executorSummary" : {
+ "driver" : {
+ "taskTime" : 533,
+ "failedTasks" : 0,
+ "succeededTasks" : 10,
+ "killedTasks" : 0,
+ "inputBytes" : 0,
+ "inputRecords" : 0,
+ "outputBytes" : 0,
+ "outputRecords" : 0,
+ "shuffleRead" : 3760,
+ "shuffleReadRecords" : 100,
+ "shuffleWrite" : 590,
+ "shuffleWriteRecords" : 10,
+ "memoryBytesSpilled" : 0,
+ "diskBytesSpilled" : 0,
+ "isBlacklistedForStage" : false,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
+ },
+ "isExcludedForStage" : false
+ }
+ },
+ "killedTasksSummary" : { },
+ "resourceProfileId" : 0,
+ "peakExecutorMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0,
+ "TotalGCTime" : 0
+ }
+} ]
\ No newline at end of file
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
index f859ab6..f32d40c 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
@@ -2,6 +2,7 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.494GMT",
"duration" : 435,
"executorId" : "driver",
@@ -51,6 +52,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.502GMT",
"duration" : 421,
"executorId" : "driver",
@@ -100,6 +102,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.503GMT",
"duration" : 419,
"executorId" : "driver",
@@ -149,6 +152,8 @@
"taskId" : 3,
"index" : 3,
"attempt" : 0,
+ "partitionId" : -1,
+"partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.504GMT",
"duration" : 423,
"executorId" : "driver",
@@ -198,6 +203,7 @@
"taskId" : 4,
"index" : 4,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.504GMT",
"duration" : 419,
"executorId" : "driver",
@@ -247,6 +253,7 @@
"taskId" : 5,
"index" : 5,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.505GMT",
"duration" : 414,
"executorId" : "driver",
@@ -296,6 +303,7 @@
"taskId" : 6,
"index" : 6,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.505GMT",
"duration" : 419,
"executorId" : "driver",
@@ -345,6 +353,7 @@
"taskId" : 7,
"index" : 7,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.506GMT",
"duration" : 423,
"executorId" : "driver",
@@ -394,6 +403,7 @@
"taskId" : 8,
"index" : 8,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.914GMT",
"duration" : 88,
"executorId" : "driver",
@@ -443,6 +453,7 @@
"taskId" : 9,
"index" : 9,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.915GMT",
"duration" : 101,
"executorId" : "driver",
@@ -492,6 +503,7 @@
"taskId" : 10,
"index" : 10,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.916GMT",
"duration" : 99,
"executorId" : "driver",
@@ -541,6 +553,7 @@
"taskId" : 11,
"index" : 11,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.918GMT",
"duration" : 89,
"executorId" : "driver",
@@ -590,6 +603,7 @@
"taskId" : 12,
"index" : 12,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.923GMT",
"duration" : 93,
"executorId" : "driver",
@@ -639,6 +653,7 @@
"taskId" : 13,
"index" : 13,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.924GMT",
"duration" : 138,
"executorId" : "driver",
@@ -688,6 +703,7 @@
"taskId" : 14,
"index" : 14,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.925GMT",
"duration" : 94,
"executorId" : "driver",
@@ -737,6 +753,7 @@
"taskId" : 15,
"index" : 15,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.928GMT",
"duration" : 83,
"executorId" : "driver",
@@ -786,6 +803,7 @@
"taskId" : 16,
"index" : 16,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.001GMT",
"duration" : 98,
"executorId" : "driver",
@@ -835,6 +853,7 @@
"taskId" : 17,
"index" : 17,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.005GMT",
"duration" : 123,
"executorId" : "driver",
@@ -884,6 +903,7 @@
"taskId" : 18,
"index" : 18,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.010GMT",
"duration" : 105,
"executorId" : "driver",
@@ -933,6 +953,7 @@
"taskId" : 19,
"index" : 19,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.012GMT",
"duration" : 94,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
index 5a0d214..2ab5903b 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
@@ -2,6 +2,7 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-16T19:25:36.515GMT",
"duration" : 61,
"executorId" : "driver",
@@ -56,6 +57,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-16T19:25:36.521GMT",
"duration" : 53,
"executorId" : "driver",
@@ -110,6 +112,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-16T19:25:36.522GMT",
"duration" : 48,
"executorId" : "driver",
@@ -164,6 +167,7 @@
"taskId" : 3,
"index" : 3,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-16T19:25:36.522GMT",
"duration" : 50,
"executorId" : "driver",
@@ -218,6 +222,7 @@
"taskId" : 4,
"index" : 4,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-16T19:25:36.522GMT",
"duration" : 52,
"executorId" : "driver",
@@ -272,6 +277,7 @@
"taskId" : 5,
"index" : 5,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-16T19:25:36.523GMT",
"duration" : 52,
"executorId" : "driver",
@@ -326,6 +332,7 @@
"taskId" : 6,
"index" : 6,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-16T19:25:36.523GMT",
"duration" : 51,
"executorId" : "driver",
@@ -380,6 +387,7 @@
"taskId" : 7,
"index" : 7,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-16T19:25:36.524GMT",
"duration" : 51,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
index fb9a169..f50a379 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
@@ -2,6 +2,7 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-17T23:12:16.515GMT",
"duration" : 61,
"executorId" : "driver",
@@ -56,6 +57,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-17T23:12:16.521GMT",
"duration" : 53,
"executorId" : "driver",
@@ -110,6 +112,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-17T23:12:16.522GMT",
"duration" : 48,
"executorId" : "driver",
@@ -164,6 +167,7 @@
"taskId" : 3,
"index" : 3,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-17T23:12:16.522GMT",
"duration" : 50,
"executorId" : "driver",
@@ -218,6 +222,7 @@
"taskId" : 4,
"index" : 4,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-17T23:12:16.522GMT",
"duration" : 52,
"executorId" : "driver",
@@ -272,6 +277,7 @@
"taskId" : 5,
"index" : 5,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-17T23:12:16.523GMT",
"duration" : 52,
"executorId" : "driver",
@@ -326,6 +332,7 @@
"taskId" : 6,
"index" : 6,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-17T23:12:16.523GMT",
"duration" : 51,
"executorId" : "driver",
@@ -380,6 +387,7 @@
"taskId" : 7,
"index" : 7,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-03-17T23:12:16.524GMT",
"duration" : 51,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
index d83528d..01500db 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
@@ -2,6 +2,7 @@
"taskId" : 10,
"index" : 10,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.916GMT",
"duration" : 99,
"executorId" : "driver",
@@ -51,6 +52,7 @@
"taskId" : 11,
"index" : 11,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.918GMT",
"duration" : 89,
"executorId" : "driver",
@@ -100,6 +102,7 @@
"taskId" : 12,
"index" : 12,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.923GMT",
"duration" : 93,
"executorId" : "driver",
@@ -149,6 +152,7 @@
"taskId" : 13,
"index" : 13,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.924GMT",
"duration" : 138,
"executorId" : "driver",
@@ -198,6 +202,7 @@
"taskId" : 14,
"index" : 14,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.925GMT",
"duration" : 94,
"executorId" : "driver",
@@ -247,6 +252,7 @@
"taskId" : 15,
"index" : 15,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.928GMT",
"duration" : 83,
"executorId" : "driver",
@@ -296,6 +302,7 @@
"taskId" : 16,
"index" : 16,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.001GMT",
"duration" : 98,
"executorId" : "driver",
@@ -345,6 +352,7 @@
"taskId" : 17,
"index" : 17,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.005GMT",
"duration" : 123,
"executorId" : "driver",
@@ -394,6 +402,7 @@
"taskId" : 18,
"index" : 18,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.010GMT",
"duration" : 105,
"executorId" : "driver",
@@ -443,6 +452,7 @@
"taskId" : 19,
"index" : 19,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.012GMT",
"duration" : 94,
"executorId" : "driver",
@@ -492,6 +502,7 @@
"taskId" : 20,
"index" : 20,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.014GMT",
"duration" : 90,
"executorId" : "driver",
@@ -541,6 +552,7 @@
"taskId" : 21,
"index" : 21,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.015GMT",
"duration" : 96,
"executorId" : "driver",
@@ -590,6 +602,7 @@
"taskId" : 22,
"index" : 22,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.018GMT",
"duration" : 101,
"executorId" : "driver",
@@ -639,6 +652,7 @@
"taskId" : 23,
"index" : 23,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.031GMT",
"duration" : 84,
"executorId" : "driver",
@@ -688,6 +702,7 @@
"taskId" : 24,
"index" : 24,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.098GMT",
"duration" : 52,
"executorId" : "driver",
@@ -737,6 +752,7 @@
"taskId" : 25,
"index" : 25,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.103GMT",
"duration" : 61,
"executorId" : "driver",
@@ -786,6 +802,7 @@
"taskId" : 26,
"index" : 26,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.105GMT",
"duration" : 52,
"executorId" : "driver",
@@ -835,6 +852,7 @@
"taskId" : 27,
"index" : 27,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.110GMT",
"duration" : 41,
"executorId" : "driver",
@@ -884,6 +902,7 @@
"taskId" : 28,
"index" : 28,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.113GMT",
"duration" : 49,
"executorId" : "driver",
@@ -933,6 +952,7 @@
"taskId" : 29,
"index" : 29,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.114GMT",
"duration" : 52,
"executorId" : "driver",
@@ -982,6 +1002,7 @@
"taskId" : 30,
"index" : 30,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.118GMT",
"duration" : 62,
"executorId" : "driver",
@@ -1031,6 +1052,7 @@
"taskId" : 31,
"index" : 31,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.127GMT",
"duration" : 74,
"executorId" : "driver",
@@ -1080,6 +1102,7 @@
"taskId" : 32,
"index" : 32,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.148GMT",
"duration" : 33,
"executorId" : "driver",
@@ -1129,6 +1152,7 @@
"taskId" : 33,
"index" : 33,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.149GMT",
"duration" : 58,
"executorId" : "driver",
@@ -1178,6 +1202,7 @@
"taskId" : 34,
"index" : 34,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.156GMT",
"duration" : 42,
"executorId" : "driver",
@@ -1227,6 +1252,7 @@
"taskId" : 35,
"index" : 35,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.161GMT",
"duration" : 50,
"executorId" : "driver",
@@ -1276,6 +1302,7 @@
"taskId" : 36,
"index" : 36,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.164GMT",
"duration" : 40,
"executorId" : "driver",
@@ -1325,6 +1352,7 @@
"taskId" : 37,
"index" : 37,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.165GMT",
"duration" : 42,
"executorId" : "driver",
@@ -1374,6 +1402,7 @@
"taskId" : 38,
"index" : 38,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.166GMT",
"duration" : 47,
"executorId" : "driver",
@@ -1423,6 +1452,7 @@
"taskId" : 39,
"index" : 39,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.180GMT",
"duration" : 32,
"executorId" : "driver",
@@ -1472,6 +1502,7 @@
"taskId" : 40,
"index" : 40,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.197GMT",
"duration" : 24,
"executorId" : "driver",
@@ -1521,6 +1552,7 @@
"taskId" : 41,
"index" : 41,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.200GMT",
"duration" : 24,
"executorId" : "driver",
@@ -1570,6 +1602,7 @@
"taskId" : 42,
"index" : 42,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.203GMT",
"duration" : 42,
"executorId" : "driver",
@@ -1619,6 +1652,7 @@
"taskId" : 43,
"index" : 43,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.204GMT",
"duration" : 39,
"executorId" : "driver",
@@ -1668,6 +1702,7 @@
"taskId" : 44,
"index" : 44,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.205GMT",
"duration" : 37,
"executorId" : "driver",
@@ -1717,6 +1752,7 @@
"taskId" : 45,
"index" : 45,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.206GMT",
"duration" : 37,
"executorId" : "driver",
@@ -1766,6 +1802,7 @@
"taskId" : 46,
"index" : 46,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.210GMT",
"duration" : 43,
"executorId" : "driver",
@@ -1815,6 +1852,7 @@
"taskId" : 47,
"index" : 47,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.212GMT",
"duration" : 33,
"executorId" : "driver",
@@ -1864,6 +1902,7 @@
"taskId" : 48,
"index" : 48,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.220GMT",
"duration" : 30,
"executorId" : "driver",
@@ -1913,6 +1952,7 @@
"taskId" : 49,
"index" : 49,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.223GMT",
"duration" : 34,
"executorId" : "driver",
@@ -1962,6 +2002,7 @@
"taskId" : 50,
"index" : 50,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.240GMT",
"duration" : 26,
"executorId" : "driver",
@@ -2011,6 +2052,7 @@
"taskId" : 51,
"index" : 51,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.242GMT",
"duration" : 21,
"executorId" : "driver",
@@ -2060,6 +2102,7 @@
"taskId" : 52,
"index" : 52,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.243GMT",
"duration" : 28,
"executorId" : "driver",
@@ -2109,6 +2152,7 @@
"taskId" : 53,
"index" : 53,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.244GMT",
"duration" : 29,
"executorId" : "driver",
@@ -2158,6 +2202,7 @@
"taskId" : 54,
"index" : 54,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.244GMT",
"duration" : 59,
"executorId" : "driver",
@@ -2207,6 +2252,7 @@
"taskId" : 55,
"index" : 55,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.246GMT",
"duration" : 30,
"executorId" : "driver",
@@ -2256,6 +2302,7 @@
"taskId" : 56,
"index" : 56,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.249GMT",
"duration" : 31,
"executorId" : "driver",
@@ -2305,6 +2352,7 @@
"taskId" : 57,
"index" : 57,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.257GMT",
"duration" : 21,
"executorId" : "driver",
@@ -2354,6 +2402,7 @@
"taskId" : 58,
"index" : 58,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.263GMT",
"duration" : 23,
"executorId" : "driver",
@@ -2403,6 +2452,7 @@
"taskId" : 59,
"index" : 59,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.265GMT",
"duration" : 23,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
index 82e339c..b1c7189 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
@@ -2,6 +2,7 @@
"taskId" : 6,
"index" : 6,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.505GMT",
"duration" : 419,
"executorId" : "driver",
@@ -51,6 +52,7 @@
"taskId" : 5,
"index" : 5,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.505GMT",
"duration" : 414,
"executorId" : "driver",
@@ -100,6 +102,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.502GMT",
"duration" : 421,
"executorId" : "driver",
@@ -149,6 +152,7 @@
"taskId" : 7,
"index" : 7,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.506GMT",
"duration" : 423,
"executorId" : "driver",
@@ -198,6 +202,7 @@
"taskId" : 4,
"index" : 4,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.504GMT",
"duration" : 419,
"executorId" : "driver",
@@ -247,6 +252,7 @@
"taskId" : 3,
"index" : 3,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.504GMT",
"duration" : 423,
"executorId" : "driver",
@@ -296,6 +302,7 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.494GMT",
"duration" : 435,
"executorId" : "driver",
@@ -345,6 +352,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.503GMT",
"duration" : 419,
"executorId" : "driver",
@@ -394,6 +402,7 @@
"taskId" : 22,
"index" : 22,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.018GMT",
"duration" : 101,
"executorId" : "driver",
@@ -443,6 +452,7 @@
"taskId" : 18,
"index" : 18,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.010GMT",
"duration" : 105,
"executorId" : "driver",
@@ -492,6 +502,7 @@
"taskId" : 17,
"index" : 17,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.005GMT",
"duration" : 123,
"executorId" : "driver",
@@ -541,6 +552,7 @@
"taskId" : 21,
"index" : 21,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.015GMT",
"duration" : 96,
"executorId" : "driver",
@@ -590,6 +602,7 @@
"taskId" : 19,
"index" : 19,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.012GMT",
"duration" : 94,
"executorId" : "driver",
@@ -639,6 +652,7 @@
"taskId" : 16,
"index" : 16,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.001GMT",
"duration" : 98,
"executorId" : "driver",
@@ -688,6 +702,7 @@
"taskId" : 9,
"index" : 9,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.915GMT",
"duration" : 101,
"executorId" : "driver",
@@ -737,6 +752,7 @@
"taskId" : 20,
"index" : 20,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.014GMT",
"duration" : 90,
"executorId" : "driver",
@@ -786,6 +802,7 @@
"taskId" : 14,
"index" : 14,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.925GMT",
"duration" : 94,
"executorId" : "driver",
@@ -835,6 +852,7 @@
"taskId" : 8,
"index" : 8,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.914GMT",
"duration" : 88,
"executorId" : "driver",
@@ -884,6 +902,7 @@
"taskId" : 12,
"index" : 12,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.923GMT",
"duration" : 93,
"executorId" : "driver",
@@ -933,6 +952,7 @@
"taskId" : 15,
"index" : 15,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.928GMT",
"duration" : 83,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
index 82e339c..b1c7189 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@@ -2,6 +2,7 @@
"taskId" : 6,
"index" : 6,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.505GMT",
"duration" : 419,
"executorId" : "driver",
@@ -51,6 +52,7 @@
"taskId" : 5,
"index" : 5,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.505GMT",
"duration" : 414,
"executorId" : "driver",
@@ -100,6 +102,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.502GMT",
"duration" : 421,
"executorId" : "driver",
@@ -149,6 +152,7 @@
"taskId" : 7,
"index" : 7,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.506GMT",
"duration" : 423,
"executorId" : "driver",
@@ -198,6 +202,7 @@
"taskId" : 4,
"index" : 4,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.504GMT",
"duration" : 419,
"executorId" : "driver",
@@ -247,6 +252,7 @@
"taskId" : 3,
"index" : 3,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.504GMT",
"duration" : 423,
"executorId" : "driver",
@@ -296,6 +302,7 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.494GMT",
"duration" : 435,
"executorId" : "driver",
@@ -345,6 +352,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.503GMT",
"duration" : 419,
"executorId" : "driver",
@@ -394,6 +402,7 @@
"taskId" : 22,
"index" : 22,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.018GMT",
"duration" : 101,
"executorId" : "driver",
@@ -443,6 +452,7 @@
"taskId" : 18,
"index" : 18,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.010GMT",
"duration" : 105,
"executorId" : "driver",
@@ -492,6 +502,7 @@
"taskId" : 17,
"index" : 17,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.005GMT",
"duration" : 123,
"executorId" : "driver",
@@ -541,6 +552,7 @@
"taskId" : 21,
"index" : 21,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.015GMT",
"duration" : 96,
"executorId" : "driver",
@@ -590,6 +602,7 @@
"taskId" : 19,
"index" : 19,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.012GMT",
"duration" : 94,
"executorId" : "driver",
@@ -639,6 +652,7 @@
"taskId" : 16,
"index" : 16,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.001GMT",
"duration" : 98,
"executorId" : "driver",
@@ -688,6 +702,7 @@
"taskId" : 9,
"index" : 9,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.915GMT",
"duration" : 101,
"executorId" : "driver",
@@ -737,6 +752,7 @@
"taskId" : 20,
"index" : 20,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.014GMT",
"duration" : 90,
"executorId" : "driver",
@@ -786,6 +802,7 @@
"taskId" : 14,
"index" : 14,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.925GMT",
"duration" : 94,
"executorId" : "driver",
@@ -835,6 +852,7 @@
"taskId" : 8,
"index" : 8,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.914GMT",
"duration" : 88,
"executorId" : "driver",
@@ -884,6 +902,7 @@
"taskId" : 12,
"index" : 12,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.923GMT",
"duration" : 93,
"executorId" : "driver",
@@ -933,6 +952,7 @@
"taskId" : 15,
"index" : 15,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.928GMT",
"duration" : 83,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
index 01eef1b..3fa4cad 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
@@ -2,6 +2,7 @@
"taskId" : 40,
"index" : 40,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.197GMT",
"duration" : 24,
"executorId" : "driver",
@@ -51,6 +52,7 @@
"taskId" : 41,
"index" : 41,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.200GMT",
"duration" : 24,
"executorId" : "driver",
@@ -100,6 +102,7 @@
"taskId" : 43,
"index" : 43,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.204GMT",
"duration" : 39,
"executorId" : "driver",
@@ -149,6 +152,7 @@
"taskId" : 57,
"index" : 57,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.257GMT",
"duration" : 21,
"executorId" : "driver",
@@ -198,6 +202,7 @@
"taskId" : 58,
"index" : 58,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.263GMT",
"duration" : 23,
"executorId" : "driver",
@@ -247,6 +252,7 @@
"taskId" : 68,
"index" : 68,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.306GMT",
"duration" : 22,
"executorId" : "driver",
@@ -296,6 +302,7 @@
"taskId" : 86,
"index" : 86,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.374GMT",
"duration" : 28,
"executorId" : "driver",
@@ -345,6 +352,7 @@
"taskId" : 32,
"index" : 32,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.148GMT",
"duration" : 33,
"executorId" : "driver",
@@ -394,6 +402,7 @@
"taskId" : 39,
"index" : 39,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.180GMT",
"duration" : 32,
"executorId" : "driver",
@@ -443,6 +452,7 @@
"taskId" : 42,
"index" : 42,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.203GMT",
"duration" : 42,
"executorId" : "driver",
@@ -492,6 +502,7 @@
"taskId" : 51,
"index" : 51,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.242GMT",
"duration" : 21,
"executorId" : "driver",
@@ -541,6 +552,7 @@
"taskId" : 59,
"index" : 59,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.265GMT",
"duration" : 23,
"executorId" : "driver",
@@ -590,6 +602,7 @@
"taskId" : 63,
"index" : 63,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.276GMT",
"duration" : 40,
"executorId" : "driver",
@@ -639,6 +652,7 @@
"taskId" : 87,
"index" : 87,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.374GMT",
"duration" : 36,
"executorId" : "driver",
@@ -688,6 +702,7 @@
"taskId" : 90,
"index" : 90,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.385GMT",
"duration" : 23,
"executorId" : "driver",
@@ -737,6 +752,7 @@
"taskId" : 99,
"index" : 99,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.426GMT",
"duration" : 22,
"executorId" : "driver",
@@ -786,6 +802,7 @@
"taskId" : 44,
"index" : 44,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.205GMT",
"duration" : 37,
"executorId" : "driver",
@@ -835,6 +852,7 @@
"taskId" : 47,
"index" : 47,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.212GMT",
"duration" : 33,
"executorId" : "driver",
@@ -884,6 +902,7 @@
"taskId" : 50,
"index" : 50,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.240GMT",
"duration" : 26,
"executorId" : "driver",
@@ -933,6 +952,7 @@
"taskId" : 52,
"index" : 52,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.243GMT",
"duration" : 28,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
index 28509e3..0cd3a45 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
@@ -2,6 +2,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.502GMT",
"duration" : 421,
"executorId" : "driver",
@@ -51,6 +52,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:06.503GMT",
"duration" : 419,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
index 01eef1b..3fa4cad 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
@@ -2,6 +2,7 @@
"taskId" : 40,
"index" : 40,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.197GMT",
"duration" : 24,
"executorId" : "driver",
@@ -51,6 +52,7 @@
"taskId" : 41,
"index" : 41,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.200GMT",
"duration" : 24,
"executorId" : "driver",
@@ -100,6 +102,7 @@
"taskId" : 43,
"index" : 43,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.204GMT",
"duration" : 39,
"executorId" : "driver",
@@ -149,6 +152,7 @@
"taskId" : 57,
"index" : 57,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.257GMT",
"duration" : 21,
"executorId" : "driver",
@@ -198,6 +202,7 @@
"taskId" : 58,
"index" : 58,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.263GMT",
"duration" : 23,
"executorId" : "driver",
@@ -247,6 +252,7 @@
"taskId" : 68,
"index" : 68,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.306GMT",
"duration" : 22,
"executorId" : "driver",
@@ -296,6 +302,7 @@
"taskId" : 86,
"index" : 86,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.374GMT",
"duration" : 28,
"executorId" : "driver",
@@ -345,6 +352,7 @@
"taskId" : 32,
"index" : 32,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.148GMT",
"duration" : 33,
"executorId" : "driver",
@@ -394,6 +402,7 @@
"taskId" : 39,
"index" : 39,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.180GMT",
"duration" : 32,
"executorId" : "driver",
@@ -443,6 +452,7 @@
"taskId" : 42,
"index" : 42,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.203GMT",
"duration" : 42,
"executorId" : "driver",
@@ -492,6 +502,7 @@
"taskId" : 51,
"index" : 51,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.242GMT",
"duration" : 21,
"executorId" : "driver",
@@ -541,6 +552,7 @@
"taskId" : 59,
"index" : 59,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.265GMT",
"duration" : 23,
"executorId" : "driver",
@@ -590,6 +602,7 @@
"taskId" : 63,
"index" : 63,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.276GMT",
"duration" : 40,
"executorId" : "driver",
@@ -639,6 +652,7 @@
"taskId" : 87,
"index" : 87,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.374GMT",
"duration" : 36,
"executorId" : "driver",
@@ -688,6 +702,7 @@
"taskId" : 90,
"index" : 90,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.385GMT",
"duration" : 23,
"executorId" : "driver",
@@ -737,6 +752,7 @@
"taskId" : 99,
"index" : 99,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.426GMT",
"duration" : 22,
"executorId" : "driver",
@@ -786,6 +802,7 @@
"taskId" : 44,
"index" : 44,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.205GMT",
"duration" : 37,
"executorId" : "driver",
@@ -835,6 +852,7 @@
"taskId" : 47,
"index" : 47,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.212GMT",
"duration" : 33,
"executorId" : "driver",
@@ -884,6 +902,7 @@
"taskId" : 50,
"index" : 50,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.240GMT",
"duration" : 26,
"executorId" : "driver",
@@ -933,6 +952,7 @@
"taskId" : 52,
"index" : 52,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2015-05-06T13:03:07.243GMT",
"duration" : 28,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
index 9896ace..d625c6f 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
@@ -2,6 +2,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2016-11-15T23:20:44.052GMT",
"duration" : 675,
"executorId" : "0",
@@ -55,6 +56,7 @@
"taskId" : 3,
"index" : 3,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2016-11-15T23:20:44.053GMT",
"duration" : 725,
"executorId" : "2",
@@ -108,6 +110,7 @@
"taskId" : 5,
"index" : 5,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2016-11-15T23:20:44.055GMT",
"duration" : 665,
"executorId" : "0",
@@ -161,6 +164,7 @@
"taskId" : 7,
"index" : 7,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2016-11-15T23:20:44.056GMT",
"duration" : 685,
"executorId" : "2",
@@ -214,6 +218,7 @@
"taskId" : 9,
"index" : 9,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2016-11-15T23:20:44.057GMT",
"duration" : 732,
"executorId" : "0",
@@ -267,6 +272,7 @@
"taskId" : 11,
"index" : 11,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2016-11-15T23:20:44.058GMT",
"duration" : 678,
"executorId" : "2",
@@ -320,6 +326,7 @@
"taskId" : 13,
"index" : 13,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2016-11-15T23:20:44.060GMT",
"duration" : 669,
"executorId" : "0",
@@ -373,6 +380,7 @@
"taskId" : 15,
"index" : 15,
"attempt" : 0,
+ "partitionId" : -1,
"launchTime" : "2016-11-15T23:20:44.065GMT",
"duration" : 672,
"executorId" : "2",
@@ -426,6 +434,7 @@
"taskId" : 19,
"index" : 11,
"attempt" : 1,
+ "partitionId": -1,
"launchTime" : "2016-11-15T23:20:44.736GMT",
"duration" : 13,
"executorId" : "2",
@@ -479,6 +488,7 @@
"taskId" : 20,
"index" : 15,
"attempt" : 1,
+ "partitionId": -1,
"launchTime" : "2016-11-15T23:20:44.737GMT",
"duration" : 19,
"executorId" : "2",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json
similarity index 58%
copy from core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
copy to core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json
index fb9a169..9d8a38c 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json
@@ -2,33 +2,29 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
- "launchTime" : "2015-03-17T23:12:16.515GMT",
- "duration" : 61,
+ "partitionId" : 0,
+ "launchTime" : "2022-01-13T02:04:56.413GMT",
+ "duration" : 480,
"executorId" : "driver",
- "host" : "localhost",
+ "host" : "172.22.200.52",
"status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
- "accumulatorUpdates" : [ {
- "id" : 1,
- "name" : "my counter",
- "update" : "78",
- "value" : "5050"
- } ],
+ "accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 14,
- "executorDeserializeCpuTime" : 0,
- "executorRunTime" : 15,
- "executorCpuTime" : 0,
- "resultSize" : 697,
- "jvmGcTime" : 0,
- "resultSerializationTime" : 2,
+ "executorDeserializeTime" : 277,
+ "executorDeserializeCpuTime" : 142074000,
+ "executorRunTime" : 158,
+ "executorCpuTime" : 48671000,
+ "resultSize" : 1836,
+ "jvmGcTime" : 11,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
- "recordsRead" : 0
+ "recordsRead" : 12
},
"outputMetrics" : {
"bytesWritten" : 0,
@@ -44,45 +40,41 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 0,
- "writeTime" : 0,
- "recordsWritten" : 0
+ "bytesWritten" : 468,
+ "writeTime" : 33968833,
+ "recordsWritten" : 12
}
},
"executorLogs" : { },
- "schedulerDelay" : 30,
+ "schedulerDelay" : 44,
"gettingResultTime" : 0
}, {
"taskId" : 1,
"index" : 1,
"attempt" : 0,
- "launchTime" : "2015-03-17T23:12:16.521GMT",
- "duration" : 53,
+ "partitionId" : 1,
+ "launchTime" : "2022-01-13T02:04:56.425GMT",
+ "duration" : 468,
"executorId" : "driver",
- "host" : "localhost",
+ "host" : "172.22.200.52",
"status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
- "accumulatorUpdates" : [ {
- "id" : 1,
- "name" : "my counter",
- "update" : "247",
- "value" : "2175"
- } ],
+ "accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 14,
- "executorDeserializeCpuTime" : 0,
- "executorRunTime" : 15,
- "executorCpuTime" : 0,
- "resultSize" : 697,
- "jvmGcTime" : 0,
- "resultSerializationTime" : 2,
+ "executorDeserializeTime" : 277,
+ "executorDeserializeCpuTime" : 141303000,
+ "executorRunTime" : 158,
+ "executorCpuTime" : 52706000,
+ "resultSize" : 1836,
+ "jvmGcTime" : 11,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
- "recordsRead" : 0
+ "recordsRead" : 13
},
"outputMetrics" : {
"bytesWritten" : 0,
@@ -98,45 +90,41 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 0,
- "writeTime" : 0,
- "recordsWritten" : 0
+ "bytesWritten" : 472,
+ "writeTime" : 32707462,
+ "recordsWritten" : 13
}
},
"executorLogs" : { },
- "schedulerDelay" : 22,
+ "schedulerDelay" : 32,
"gettingResultTime" : 0
}, {
"taskId" : 2,
"index" : 2,
"attempt" : 0,
- "launchTime" : "2015-03-17T23:12:16.522GMT",
- "duration" : 48,
+ "partitionId" : 2,
+ "launchTime" : "2022-01-13T02:04:56.425GMT",
+ "duration" : 467,
"executorId" : "driver",
- "host" : "localhost",
+ "host" : "172.22.200.52",
"status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
- "accumulatorUpdates" : [ {
- "id" : 1,
- "name" : "my counter",
- "update" : "378",
- "value" : "378"
- } ],
+ "accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 13,
- "executorDeserializeCpuTime" : 0,
- "executorRunTime" : 15,
- "executorCpuTime" : 0,
- "resultSize" : 697,
- "jvmGcTime" : 0,
- "resultSerializationTime" : 2,
+ "executorDeserializeTime" : 277,
+ "executorDeserializeCpuTime" : 144165000,
+ "executorRunTime" : 158,
+ "executorCpuTime" : 47904000,
+ "resultSize" : 1836,
+ "jvmGcTime" : 11,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
- "recordsRead" : 0
+ "recordsRead" : 12
},
"outputMetrics" : {
"bytesWritten" : 0,
@@ -152,45 +140,41 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 0,
- "writeTime" : 0,
- "recordsWritten" : 0
+ "bytesWritten" : 468,
+ "writeTime" : 31530251,
+ "recordsWritten" : 12
}
},
"executorLogs" : { },
- "schedulerDelay" : 18,
+ "schedulerDelay" : 31,
"gettingResultTime" : 0
}, {
"taskId" : 3,
"index" : 3,
"attempt" : 0,
- "launchTime" : "2015-03-17T23:12:16.522GMT",
- "duration" : 50,
+ "partitionId" : 3,
+ "launchTime" : "2022-01-13T02:04:56.425GMT",
+ "duration" : 468,
"executorId" : "driver",
- "host" : "localhost",
+ "host" : "172.22.200.52",
"status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
- "accumulatorUpdates" : [ {
- "id" : 1,
- "name" : "my counter",
- "update" : "572",
- "value" : "950"
- } ],
+ "accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 13,
- "executorDeserializeCpuTime" : 0,
- "executorRunTime" : 15,
- "executorCpuTime" : 0,
- "resultSize" : 697,
- "jvmGcTime" : 0,
- "resultSerializationTime" : 2,
+ "executorDeserializeTime" : 277,
+ "executorDeserializeCpuTime" : 81219000,
+ "executorRunTime" : 158,
+ "executorCpuTime" : 50624000,
+ "resultSize" : 1836,
+ "jvmGcTime" : 11,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
- "recordsRead" : 0
+ "recordsRead" : 13
},
"outputMetrics" : {
"bytesWritten" : 0,
@@ -206,45 +190,41 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 0,
- "writeTime" : 0,
- "recordsWritten" : 0
+ "bytesWritten" : 472,
+ "writeTime" : 33237160,
+ "recordsWritten" : 13
}
},
"executorLogs" : { },
- "schedulerDelay" : 20,
+ "schedulerDelay" : 32,
"gettingResultTime" : 0
}, {
"taskId" : 4,
"index" : 4,
"attempt" : 0,
- "launchTime" : "2015-03-17T23:12:16.522GMT",
- "duration" : 52,
+ "partitionId" : 4,
+ "launchTime" : "2022-01-13T02:04:56.426GMT",
+ "duration" : 466,
"executorId" : "driver",
- "host" : "localhost",
+ "host" : "172.22.200.52",
"status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
- "accumulatorUpdates" : [ {
- "id" : 1,
- "name" : "my counter",
- "update" : "678",
- "value" : "2853"
- } ],
+ "accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
- "executorDeserializeCpuTime" : 0,
- "executorRunTime" : 15,
- "executorCpuTime" : 0,
- "resultSize" : 697,
- "jvmGcTime" : 0,
+ "executorDeserializeTime" : 277,
+ "executorDeserializeCpuTime" : 143825000,
+ "executorRunTime" : 158,
+ "executorCpuTime" : 42837000,
+ "resultSize" : 1836,
+ "jvmGcTime" : 11,
"resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
- "recordsRead" : 0
+ "recordsRead" : 12
},
"outputMetrics" : {
"bytesWritten" : 0,
@@ -260,45 +240,41 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 0,
- "writeTime" : 0,
- "recordsWritten" : 0
+ "bytesWritten" : 468,
+ "writeTime" : 32008329,
+ "recordsWritten" : 12
}
},
"executorLogs" : { },
- "schedulerDelay" : 24,
+ "schedulerDelay" : 30,
"gettingResultTime" : 0
}, {
"taskId" : 5,
"index" : 5,
"attempt" : 0,
- "launchTime" : "2015-03-17T23:12:16.523GMT",
- "duration" : 52,
+ "partitionId" : 5,
+ "launchTime" : "2022-01-13T02:04:56.426GMT",
+ "duration" : 465,
"executorId" : "driver",
- "host" : "localhost",
+ "host" : "172.22.200.52",
"status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
- "accumulatorUpdates" : [ {
- "id" : 1,
- "name" : "my counter",
- "update" : "897",
- "value" : "3750"
- } ],
+ "accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
- "executorDeserializeCpuTime" : 0,
- "executorRunTime" : 15,
- "executorCpuTime" : 0,
- "resultSize" : 697,
- "jvmGcTime" : 0,
- "resultSerializationTime" : 2,
+ "executorDeserializeTime" : 277,
+ "executorDeserializeCpuTime" : 156363000,
+ "executorRunTime" : 158,
+ "executorCpuTime" : 60658000,
+ "resultSize" : 1836,
+ "jvmGcTime" : 11,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
- "recordsRead" : 0
+ "recordsRead" : 13
},
"outputMetrics" : {
"bytesWritten" : 0,
@@ -314,45 +290,41 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 0,
- "writeTime" : 0,
- "recordsWritten" : 0
+ "bytesWritten" : 472,
+ "writeTime" : 31370834,
+ "recordsWritten" : 13
}
},
"executorLogs" : { },
- "schedulerDelay" : 23,
+ "schedulerDelay" : 29,
"gettingResultTime" : 0
}, {
"taskId" : 6,
"index" : 6,
"attempt" : 0,
- "launchTime" : "2015-03-17T23:12:16.523GMT",
- "duration" : 51,
+ "partitionId" : 6,
+ "launchTime" : "2022-01-13T02:04:56.427GMT",
+ "duration" : 461,
"executorId" : "driver",
- "host" : "localhost",
+ "host" : "172.22.200.52",
"status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
- "accumulatorUpdates" : [ {
- "id" : 1,
- "name" : "my counter",
- "update" : "978",
- "value" : "1928"
- } ],
+ "accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
- "executorDeserializeCpuTime" : 0,
- "executorRunTime" : 15,
- "executorCpuTime" : 0,
- "resultSize" : 697,
- "jvmGcTime" : 0,
- "resultSerializationTime" : 2,
+ "executorDeserializeTime" : 277,
+ "executorDeserializeCpuTime" : 131261000,
+ "executorRunTime" : 158,
+ "executorCpuTime" : 75295000,
+ "resultSize" : 1836,
+ "jvmGcTime" : 11,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
- "recordsRead" : 0
+ "recordsRead" : 12
},
"outputMetrics" : {
"bytesWritten" : 0,
@@ -368,45 +340,41 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 0,
- "writeTime" : 0,
- "recordsWritten" : 0
+ "bytesWritten" : 468,
+ "writeTime" : 32402787,
+ "recordsWritten" : 12
}
},
"executorLogs" : { },
- "schedulerDelay" : 22,
+ "schedulerDelay" : 25,
"gettingResultTime" : 0
}, {
"taskId" : 7,
"index" : 7,
"attempt" : 0,
- "launchTime" : "2015-03-17T23:12:16.524GMT",
- "duration" : 51,
+ "partitionId" : 7,
+ "launchTime" : "2022-01-13T02:04:56.427GMT",
+ "duration" : 466,
"executorId" : "driver",
- "host" : "localhost",
+ "host" : "172.22.200.52",
"status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
- "accumulatorUpdates" : [ {
- "id" : 1,
- "name" : "my counter",
- "update" : "1222",
- "value" : "4972"
- } ],
+ "accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
- "executorDeserializeCpuTime" : 0,
- "executorRunTime" : 15,
- "executorCpuTime" : 0,
- "resultSize" : 697,
- "jvmGcTime" : 0,
- "resultSerializationTime" : 2,
+ "executorDeserializeTime" : 277,
+ "executorDeserializeCpuTime" : 145865000,
+ "executorRunTime" : 158,
+ "executorCpuTime" : 42833000,
+ "resultSize" : 1836,
+ "jvmGcTime" : 11,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
- "recordsRead" : 0
+ "recordsRead" : 13
},
"outputMetrics" : {
"bytesWritten" : 0,
@@ -422,12 +390,12 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 0,
- "writeTime" : 0,
- "recordsWritten" : 0
+ "bytesWritten" : 472,
+ "writeTime" : 32328957,
+ "recordsWritten" : 13
}
},
"executorLogs" : { },
- "schedulerDelay" : 22,
+ "schedulerDelay" : 30,
"gettingResultTime" : 0
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index 281f7b3..9a822b0 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -50,6 +50,7 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-03-16T19:25:36.515GMT",
"duration" : 61,
"executorId" : "driver",
@@ -105,6 +106,7 @@
"taskId" : 5,
"index" : 5,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-03-16T19:25:36.523GMT",
"duration" : 52,
"executorId" : "driver",
@@ -160,6 +162,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-03-16T19:25:36.521GMT",
"duration" : 53,
"executorId" : "driver",
@@ -215,6 +218,7 @@
"taskId" : 6,
"index" : 6,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-03-16T19:25:36.523GMT",
"duration" : 51,
"executorId" : "driver",
@@ -270,6 +274,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-03-16T19:25:36.522GMT",
"duration" : 48,
"executorId" : "driver",
@@ -325,6 +330,7 @@
"taskId" : 7,
"index" : 7,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-03-16T19:25:36.524GMT",
"duration" : 51,
"executorId" : "driver",
@@ -380,6 +386,7 @@
"taskId" : 3,
"index" : 3,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-03-16T19:25:36.522GMT",
"duration" : 50,
"executorId" : "driver",
@@ -435,6 +442,7 @@
"taskId" : 4,
"index" : 4,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2015-03-16T19:25:36.522GMT",
"duration" : 52,
"executorId" : "driver",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
index 5b2c205..2ded2de 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
@@ -46,6 +46,7 @@
"taskId" : 42,
"index" : 10,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.120GMT",
"duration" : 1923,
"executorId" : "0",
@@ -99,6 +100,7 @@
"taskId" : 37,
"index" : 5,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.100GMT",
"duration" : 1915,
"executorId" : "0",
@@ -152,6 +154,7 @@
"taskId" : 46,
"index" : 14,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.132GMT",
"duration" : 1905,
"executorId" : "0",
@@ -205,6 +208,7 @@
"taskId" : 38,
"index" : 6,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.104GMT",
"duration" : 1835,
"executorId" : "0",
@@ -258,6 +262,7 @@
"taskId" : 33,
"index" : 1,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.082GMT",
"duration" : 1943,
"executorId" : "0",
@@ -311,6 +316,7 @@
"taskId" : 41,
"index" : 9,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.116GMT",
"duration" : 1916,
"executorId" : "0",
@@ -364,6 +370,7 @@
"taskId" : 32,
"index" : 0,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.077GMT",
"duration" : 1960,
"executorId" : "0",
@@ -417,6 +424,7 @@
"taskId" : 34,
"index" : 2,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.087GMT",
"duration" : 1939,
"executorId" : "0",
@@ -470,6 +478,7 @@
"taskId" : 45,
"index" : 13,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.129GMT",
"duration" : 1895,
"executorId" : "0",
@@ -523,6 +532,7 @@
"taskId" : 44,
"index" : 12,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.126GMT",
"duration" : 1917,
"executorId" : "0",
@@ -576,6 +586,7 @@
"taskId" : 39,
"index" : 7,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.109GMT",
"duration" : 1915,
"executorId" : "0",
@@ -629,6 +640,7 @@
"taskId" : 35,
"index" : 3,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.091GMT",
"duration" : 1925,
"executorId" : "0",
@@ -682,6 +694,7 @@
"taskId" : 43,
"index" : 11,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.123GMT",
"duration" : 1906,
"executorId" : "0",
@@ -735,6 +748,7 @@
"taskId" : 40,
"index" : 8,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.112GMT",
"duration" : 1904,
"executorId" : "0",
@@ -788,6 +802,7 @@
"taskId" : 36,
"index" : 4,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.095GMT",
"duration" : 1920,
"executorId" : "0",
@@ -841,6 +856,7 @@
"taskId" : 47,
"index" : 15,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.136GMT",
"duration" : 1878,
"executorId" : "0",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
index 32cf571..f4c2182 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
@@ -46,6 +46,7 @@
"taskId" : 0,
"index" : 0,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2021-08-10T23:27:53.885GMT",
"duration" : 2234,
"executorId" : "7",
@@ -99,6 +100,7 @@
"taskId" : 1,
"index" : 1,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2021-08-10T23:27:53.903GMT",
"duration" : 2647,
"executorId" : "5",
@@ -152,6 +154,7 @@
"taskId" : 2,
"index" : 2,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2021-08-10T23:27:53.904GMT",
"duration" : 5124,
"executorId" : "8",
@@ -205,6 +208,7 @@
"taskId" : 3,
"index" : 3,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2021-08-10T23:27:53.904GMT",
"duration" : 63773,
"executorId" : "10",
@@ -258,6 +262,7 @@
"taskId" : 4,
"index" : 3,
"attempt" : 1,
+ "partitionId": -1,
"launchTime" : "2021-08-10T23:28:04.499GMT",
"duration" : 53201,
"executorId" : "7",
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
index 3be20df..164395f 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
@@ -46,6 +46,7 @@
"taskId" : 42,
"index" : 10,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.120GMT",
"duration" : 1923,
"executorId" : "0",
@@ -99,6 +100,7 @@
"taskId" : 37,
"index" : 5,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.100GMT",
"duration" : 1915,
"executorId" : "0",
@@ -152,6 +154,7 @@
"taskId" : 46,
"index" : 14,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.132GMT",
"duration" : 1905,
"executorId" : "0",
@@ -205,6 +208,7 @@
"taskId" : 38,
"index" : 6,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.104GMT",
"duration" : 1835,
"executorId" : "0",
@@ -258,6 +262,7 @@
"taskId" : 33,
"index" : 1,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.082GMT",
"duration" : 1943,
"executorId" : "0",
@@ -311,6 +316,7 @@
"taskId" : 41,
"index" : 9,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.116GMT",
"duration" : 1916,
"executorId" : "0",
@@ -364,6 +370,7 @@
"taskId" : 32,
"index" : 0,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.077GMT",
"duration" : 1960,
"executorId" : "0",
@@ -417,6 +424,7 @@
"taskId" : 34,
"index" : 2,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.087GMT",
"duration" : 1939,
"executorId" : "0",
@@ -470,6 +478,7 @@
"taskId" : 45,
"index" : 13,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.129GMT",
"duration" : 1895,
"executorId" : "0",
@@ -523,6 +532,7 @@
"taskId" : 44,
"index" : 12,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.126GMT",
"duration" : 1917,
"executorId" : "0",
@@ -576,6 +586,7 @@
"taskId" : 39,
"index" : 7,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.109GMT",
"duration" : 1915,
"executorId" : "0",
@@ -629,6 +640,7 @@
"taskId" : 35,
"index" : 3,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.091GMT",
"duration" : 1925,
"executorId" : "0",
@@ -682,6 +694,7 @@
"taskId" : 43,
"index" : 11,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.123GMT",
"duration" : 1906,
"executorId" : "0",
@@ -735,6 +748,7 @@
"taskId" : 40,
"index" : 8,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.112GMT",
"duration" : 1904,
"executorId" : "0",
@@ -788,6 +802,7 @@
"taskId" : 36,
"index" : 4,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.095GMT",
"duration" : 1920,
"executorId" : "0",
@@ -841,6 +856,7 @@
"taskId" : 47,
"index" : 15,
"attempt" : 0,
+ "partitionId": -1,
"launchTime" : "2020-07-07T03:11:21.136GMT",
"duration" : 1878,
"executorId" : "0",
diff --git a/core/src/test/resources/spark-events/local-1642039451826 b/core/src/test/resources/spark-events/local-1642039451826
new file mode 100644
index 0000000..d4bf1c8
--- /dev/null
+++ b/core/src/test/resources/spark-events/local-1642039451826
@@ -0,0 +1,115 @@
+{"Event":"SparkListenerLogStart","Spark Version":"3.3.0-SNAPSHOT"}
+{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
+{"Event":"SparkListenerExecutorAdded","Timestamp":1642039451891,"Executor ID":"driver","Executor Info":{"Host":"172.22.200.52","Total Cores":8,"Log Urls":{},"Attributes":{},"Resources":{},"Resource Profile Id":0}}
+{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"172.22.200.52","Port":61039},"Maximum Memory":384093388,"Timestamp":1642039451909,"Maximum Onheap Memory":384093388,"Maximum Offheap Memory":0}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/Library/Java/JavaVirtualMachines/zulu-8.jdk/Contents/Home/jre","Java Version":"1.8.0_312 (Azul Systems, Inc.)","Scala Version":"version 2.12.15"},"Spark Properties":{"spark.executor.extraJavaOptions":"-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UN [...]
+{"Event":"SparkListenerApplicationStart","App Name":"Spark shell","App ID":"local-1642039451826","Timestamp":1642039450519,"User":"lijunqing"}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"count at <console>:23","details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3130)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:23)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:27)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:29)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:31)\n$line15.$read$$iw$$iw$$iw$$iw.<init>(<console>: [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (12)\n+- == Current Plan ==\n HashAggregate (7)\n +- Exchange (6)\n +- HashAggregate (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * Project (2)\n +- * Range (1)\n+- == Initial Plan ==\n HashAggregate (11)\n +- Exchange (10)\n +- HashAggregate (9)\n [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[62,10]]}
+{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1642039496191,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":8,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partiti [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":8,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":8,"Number of Cached Partitions":0, [...]
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039496413,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1642039496425,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1642039496425,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1642039496425,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1642039496426,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1642039496426,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1642039496427,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1642039496427,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1642039496427,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039496888,"Failed":false,"Killed":false,"Accumulables":[{"ID":36,"Name":"number of output rows","Update":"12","Value":"12","Interna [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1642039496426,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039496891,"Failed":false,"Killed":false,"Accumulables":[{"ID":36,"Name":"number of output rows","Update":"13","Value":"25","Interna [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1642039496426,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039496892,"Failed":false,"Killed":false,"Accumulables":[{"ID":36,"Name":"number of output rows","Update":"12","Value":"37","Interna [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1642039496425,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039496892,"Failed":false,"Killed":false,"Accumulables":[{"ID":36,"Name":"number of output rows","Update":"12","Value":"49","Interna [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039496413,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039496893,"Failed":false,"Killed":false,"Accumulables":[{"ID":36,"Name":"number of output rows","Update":"12","Value":"61","Interna [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1642039496425,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039496893,"Failed":false,"Killed":false,"Accumulables":[{"ID":36,"Name":"number of output rows","Update":"13","Value":"74","Interna [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1642039496427,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039496893,"Failed":false,"Killed":false,"Accumulables":[{"ID":36,"Name":"number of output rows","Update":"13","Value":"87","Interna [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1642039496425,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039496893,"Failed":false,"Killed":false,"Accumulables":[{"ID":36,"Name":"number of output rows","Update":"13","Value":"100","Intern [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":8,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":8,"Number of Cached Partitions":0, [...]
+{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1642039496914,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (13)\n+- == Current Plan ==\n HashAggregate (8)\n +- ShuffleQueryStage (7)\n +- Exchange (6)\n +- * HashAggregate (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * Project (2)\n +- * Range (1)\n+- == Initial Plan ==\n HashAggregate (12)\n [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[106,1]]}
+{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1642039497010,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":8,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partiti [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":10,"RDD Info":[{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":10,"Number of Cached Partitions":0, [...]
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039497053,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1642039497055,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1642039497055,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1642039497055,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1642039497056,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1642039497056,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1642039497056,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1642039497056,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1642039497114,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1642039497115,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1642039497056,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497115,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"8","Internal":true, [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1642039497055,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497116,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"16","Internal":true [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1642039497056,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497118,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"24","Internal":true [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1642039497056,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497118,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"32","Internal":true [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1642039497055,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497119,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"40","Internal":true [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1642039497055,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497120,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"48","Internal":true, [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1642039497056,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497123,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"56","Internal":true [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039497053,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497124,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"64","Internal":true, [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1642039497115,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497125,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"72","Internal":true [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1642039497114,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497126,"Failed":false,"Killed":false,"Accumulables":[{"ID":64,"Name":"local blocks read","Update":"8","Value":"80","Internal":true [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":10,"RDD Info":[{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":10,"Number of Cached Partitions":0, [...]
+{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1642039497127,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (13)\n+- == Final Plan ==\n * HashAggregate (8)\n +- ShuffleQueryStage (7)\n +- Exchange (6)\n +- * HashAggregate (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * Project (2)\n +- * Range (1)\n+- == Initial Plan ==\n HashAggregate (12)\n [...]
+{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1642039497179,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":1,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at <console>:23","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Numb [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":1,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at <console>:23","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached P [...]
+{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039497187,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039497187,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039497248,"Failed":false,"Killed":false,"Accumulables":[{"ID":108,"Name":"local blocks read","Update":"10","Value":"10","Internal":true," [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":1,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at <console>:23","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached P [...]
+{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1642039497251,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1642039497259}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"count at <console>:23","details":"org.apache.spark.sql.Dataset.count(Dataset.scala:3130)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:23)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:27)\n$line16.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:29)\n$line16.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:31)\n$line16.$read$$iw$$iw$$iw$$iw.<init>(<console>: [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (12)\n+- == Current Plan ==\n HashAggregate (7)\n +- Exchange (6)\n +- HashAggregate (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * Project (2)\n +- * Range (1)\n+- == Initial Plan ==\n HashAggregate (11)\n +- Exchange (10)\n +- HashAggregate (9)\n [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[243,10]]}
+{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1642039528323,"Stage Infos":[{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":5,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Part [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":5,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":5,"Number of Cached Partitions" [...]
+{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039528357,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1642039528358,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1642039528358,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1642039528358,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1642039528358,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039528357,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528383,"Failed":false,"Killed":false,"Accumulables":[{"ID":217,"Name":"number of output rows","Update":"51","Value":"51","Inter [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1642039528358,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528384,"Failed":false,"Killed":false,"Accumulables":[{"ID":217,"Name":"number of output rows","Update":"52","Value":"103","Inte [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1642039528358,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528384,"Failed":false,"Killed":false,"Accumulables":[{"ID":217,"Name":"number of output rows","Update":"52","Value":"155","Inte [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1642039528358,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528386,"Failed":false,"Killed":false,"Accumulables":[{"ID":217,"Name":"number of output rows","Update":"52","Value":"207","Inte [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1642039528358,"Executor ID":"driver","Host":"172.22.200.52","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528388,"Failed":false,"Killed":false,"Accumulables":[{"ID":217,"Name":"number of output rows","Update":"51","Value":"258","Inte [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":5,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":5,"Number of Cached Partitions" [...]
+{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1642039528389,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (13)\n+- == Current Plan ==\n HashAggregate (8)\n +- ShuffleQueryStage (7)\n +- Exchange (6)\n +- * HashAggregate (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * Project (2)\n +- * Range (1)\n+- == Initial Plan ==\n HashAggregate (12)\n [...]
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[287,1]]}
+{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1642039528402,"Stage Infos":[{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":5,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Part [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":10,"RDD Info":[{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"32\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":10,"Number of Cached Partitions" [...]
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039528410,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1642039528410,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1642039528425,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskStart","Stage ID":8,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1642039528426,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1642039528410,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528427,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"5","Internal":true [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528427,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"10","Internal":tru [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039528410,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528427,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"15","Internal":tru [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Partition ID":7,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528432,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"20","Internal":tru [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528432,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"25","Internal":tru [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528433,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"30","Internal":tru [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528435,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"35","Internal":tru [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":8,"Attempt":0,"Partition ID":8,"Launch Time":1642039528425,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528435,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"40","Internal":tru [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1642039528411,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528436,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"45","Internal":tru [...]
+{"Event":"SparkListenerTaskEnd","Stage ID":8,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":9,"Attempt":0,"Partition ID":9,"Launch Time":1642039528426,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528437,"Failed":false,"Killed":false,"Accumulables":[{"ID":245,"Name":"local blocks read","Update":"5","Value":"50","Internal":tru [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":10,"RDD Info":[{"RDD ID":18,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"32\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":10,"Number of Cached Partitions" [...]
+{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1642039528439,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (13)\n+- == Final Plan ==\n * HashAggregate (8)\n +- ShuffleQueryStage (7)\n +- Exchange (6)\n +- * HashAggregate (5)\n +- ShuffleQueryStage (4)\n +- Exchange (3)\n +- * Project (2)\n +- * Range (1)\n+- == Initial Plan ==\n HashAggregate (12)\n [...]
+{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1642039528461,"Stage Infos":[{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":5,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","Callsite":"count at <console>:23","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Part [...]
+{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":1,"RDD Info":[{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"43\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at <console>:23","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached [...]
+{"Event":"SparkListenerTaskStart","Stage ID":11,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039528469,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
+{"Event":"SparkListenerTaskEnd","Stage ID":11,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1642039528469,"Executor ID":"driver","Host":"172.22.200.52","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1642039528478,"Failed":false,"Killed":false,"Accumulables":[{"ID":289,"Name":"local blocks read","Update":"10","Value":"10","Internal":true, [...]
+{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"count at <console>:23","Number of Tasks":1,"RDD Info":[{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"43\",\"name\":\"mapPartitionsInternal\"}","Callsite":"count at <console>:23","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached [...]
+{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1642039528479,"Job Result":{"Result":"JobSucceeded"}}
+{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1642039528481}
+{"Event":"SparkListenerApplicationEnd","Timestamp":1642039536564}
diff --git a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
index 2fb5140..c616c43 100644
--- a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
@@ -1756,12 +1756,12 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
}
private def onExecutorBusy(manager: ExecutorAllocationManager, id: String): Unit = {
- val info = new TaskInfo(1, 1, 1, 0, id, "foo.example.com", TaskLocality.PROCESS_LOCAL, false)
+ val info = new TaskInfo(1, 1, 1, 1, 0, id, "foo.example.com", TaskLocality.PROCESS_LOCAL, false)
post(SparkListenerTaskStart(1, 1, info))
}
private def onExecutorIdle(manager: ExecutorAllocationManager, id: String): Unit = {
- val info = new TaskInfo(1, 1, 1, 0, id, "foo.example.com", TaskLocality.PROCESS_LOCAL, false)
+ val info = new TaskInfo(1, 1, 1, 1, 0, id, "foo.example.com", TaskLocality.PROCESS_LOCAL, false)
info.markFinished(TaskState.FINISHED, 1)
post(SparkListenerTaskEnd(1, 1, "foo", Success, info, new ExecutorMetrics, null))
}
@@ -1815,7 +1815,8 @@ private object ExecutorAllocationManagerSuite extends PrivateMethodTester {
taskIndex: Int,
executorId: String,
speculative: Boolean = false): TaskInfo = {
- new TaskInfo(taskId, taskIndex, 0, 0, executorId, "", TaskLocality.ANY, speculative)
+ new TaskInfo(taskId, taskIndex, 0, partitionId = taskIndex,
+ 0, executorId, "", TaskLocality.ANY, speculative)
}
/* ------------------------------------------------------- *
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index c3bd4d8..25f962a 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -146,6 +146,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
"one stage attempt json" -> "applications/local-1422981780767/stages/1/0",
"one stage attempt json details with failed task" ->
"applications/local-1422981780767/stages/1/0?details=true&taskStatus=failed",
+ "one stage json with partitionId" -> "applications/local-1642039451826/stages/2",
"stage task summary w shuffle write"
-> "applications/local-1430917381534/stages/0/0/taskSummary",
@@ -169,6 +170,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
"applications/local-1430917381534/stages/0/0/taskList?status=success&offset=1&length=2",
"stage task list w/ status & sortBy short names: runtime" ->
"applications/local-1430917381534/stages/0/0/taskList?status=success&sortBy=runtime",
+ "stage task list with partitionId" -> "applications/local-1642039451826/stages/0/0/taskList",
"stage list with accumulable json" -> "applications/local-1426533911241/1/stages",
"stage with accumulable json" -> "applications/local-1426533911241/1/stages/0/0",
diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 023e352..c61cfe6 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -2082,7 +2082,8 @@ class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with Ti
// result task 0.0 succeed
runEvent(makeCompletionEvent(taskSets(1).tasks(0), Success, 42))
// speculative result task 1.1 fetch failed
- val info = new TaskInfo(4, index = 1, attemptNumber = 1, 0L, "", "", TaskLocality.ANY, true)
+ val info = new TaskInfo(
+ 4, index = 1, attemptNumber = 1, partitionId = 1, 0L, "", "", TaskLocality.ANY, true)
runEvent(makeCompletionEvent(
taskSets(1).tasks(1),
FetchFailed(makeBlockManagerId("hostA"), shuffleDep.shuffleId, 0L, 0, 1, "ignored"),
@@ -4371,13 +4372,13 @@ class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with Ti
// Nothing in this test should break if the task info's fields are null, but
// OutputCommitCoordinator requires the task info itself to not be null.
private def createFakeTaskInfo(): TaskInfo = {
- val info = new TaskInfo(0, 0, 0, 0L, "", "", TaskLocality.ANY, false)
+ val info = new TaskInfo(0, 0, 0, 0, 0L, "", "", TaskLocality.ANY, false)
info.finishTime = 1
info
}
private def createFakeTaskInfoWithId(taskId: Long): TaskInfo = {
- val info = new TaskInfo(taskId, 0, 0, 0L, "", "", TaskLocality.ANY, false)
+ val info = new TaskInfo(taskId, 0, 0, 0, 0L, "", "", TaskLocality.ANY, false)
info.finishTime = 1
info
}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
index edb2095..d7cd63a 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
@@ -599,7 +599,8 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit
stageId: Int,
taskType: String,
executorMetrics: ExecutorMetrics): SparkListenerTaskEnd = {
- val taskInfo = new TaskInfo(taskId, taskIndex, 0, 1553291556000L, executorId, "executor",
+ val taskInfo = new TaskInfo(
+ taskId, taskIndex, 0, partitionId = taskIndex, 1553291556000L, executorId, "executor",
TaskLocality.NODE_LOCAL, false)
val taskMetrics = TaskMetrics.empty
SparkListenerTaskEnd(stageId, 0, taskType, Success, taskInfo, executorMetrics, taskMetrics)
diff --git a/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
index 336198b..39e1470 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
@@ -493,7 +493,8 @@ class ExecutorMonitorSuite extends SparkFunSuite {
speculative: Boolean = false,
duration: Long = -1L): TaskInfo = {
val start = if (duration > 0) clock.getTimeMillis() - duration else clock.getTimeMillis()
- val task = new TaskInfo(id, id, 1, start, execId, "foo.example.com",
+ val task = new TaskInfo(
+ id, id, 1, id, start, execId, "foo.example.com",
TaskLocality.PROCESS_LOCAL, speculative)
if (duration > 0) {
task.markFinished(TaskState.FINISHED, math.max(1, clock.getTimeMillis()))
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index 97a59f8..105b447 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -1860,7 +1860,8 @@ abstract class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter
private def newAttempt(orig: TaskInfo, nextId: Long): TaskInfo = {
// Task reattempts have a different ID, but the same index as the original.
- new TaskInfo(nextId, orig.index, orig.attemptNumber + 1, time, orig.executorId,
+ new TaskInfo(
+ nextId, orig.index, orig.attemptNumber + 1, orig.partitionId, time, orig.executorId,
s"${orig.executorId}.example.com", TaskLocality.PROCESS_LOCAL, orig.speculative)
}
@@ -1868,7 +1869,9 @@ abstract class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter
(1 to count).map { id =>
val exec = execs(id.toInt % execs.length)
val taskId = nextTaskId()
- new TaskInfo(taskId, taskId.toInt, 1, time, exec, s"$exec.example.com",
+ val taskIndex = id - 1
+ val partitionId = taskIndex
+ new TaskInfo(taskId, taskIndex, 1, partitionId, time, exec, s"$exec.example.com",
TaskLocality.PROCESS_LOCAL, id % 2 == 0)
}
}
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
index 53b0131..7085216 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
@@ -219,7 +219,8 @@ class AppStatusStoreSuite extends SparkFunSuite {
resourceProfileId = ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
(1 to 2).foreach {
taskId =>
- val taskInfo = new TaskInfo(taskId, taskId, 0, 0, "0", "localhost", TaskLocality.ANY,
+ val taskInfo = new TaskInfo(
+ taskId, taskId, 0, taskId, 0, "0", "localhost", TaskLocality.ANY,
false)
listener.onStageSubmitted(SparkListenerStageSubmitted(stageInfo))
listener.onTaskStart(SparkListenerTaskStart(0, 0, taskInfo))
@@ -246,14 +247,15 @@ class AppStatusStoreSuite extends SparkFunSuite {
private def newTaskData(i: Int, status: String = "SUCCESS"): TaskDataWrapper = {
new TaskDataWrapper(
- i.toLong, i, i, i, i, i, i.toString, i.toString, status, i.toString, false, Nil, None, true,
+ i.toLong, i, i, i, i, i, i,
+ i.toString, i.toString, status, i.toString, false, Nil, None, true,
i, i, i, i, i, i, i, i, i, i,
i, i, i, i, i, i, i, i, i, i,
i, i, i, i, stageId, attemptId)
}
private def writeTaskDataToStore(i: Int, store: KVStore, status: String): Unit = {
- val liveTask = new LiveTask(new TaskInfo( i.toLong, i, i, i.toLong, i.toString,
+ val liveTask = new LiveTask(new TaskInfo( i.toLong, i, i, i, i.toLong, i.toString,
i.toString, TaskLocality.ANY, false), stageId, attemptId, None)
if (status == "SUCCESS") {
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
index a01b24d..da14dcd 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
@@ -28,6 +28,7 @@ class AppStatusUtilsSuite extends SparkFunSuite {
taskId = 0,
index = 0,
attempt = 0,
+ partitionId = 0,
launchTime = new Date(1L),
resultFetchStart = None,
duration = Some(100L),
@@ -62,6 +63,7 @@ class AppStatusUtilsSuite extends SparkFunSuite {
taskId = 0,
index = 0,
attempt = 0,
+ partitionId = 0,
launchTime = new Date(1L),
resultFetchStart = None,
duration = Some(100L),
diff --git a/core/src/test/scala/org/apache/spark/status/ListenerEventsTestHelper.scala b/core/src/test/scala/org/apache/spark/status/ListenerEventsTestHelper.scala
index 99c0d95..f93c2bc 100644
--- a/core/src/test/scala/org/apache/spark/status/ListenerEventsTestHelper.scala
+++ b/core/src/test/scala/org/apache/spark/status/ListenerEventsTestHelper.scala
@@ -73,7 +73,7 @@ object ListenerEventsTestHelper {
def createTasks(ids: Seq[Long], execs: Array[String], time: Long): Seq[TaskInfo] = {
ids.zipWithIndex.map { case (id, idx) =>
val exec = execs(idx % execs.length)
- new TaskInfo(id, idx, 1, time, exec, s"$exec.example.com",
+ new TaskInfo(id, idx, 1, idx, time, exec, s"$exec.example.com",
TaskLocality.PROCESS_LOCAL, idx % 2 == 0)
}
}
@@ -84,7 +84,8 @@ object ListenerEventsTestHelper {
def createTaskWithNewAttempt(orig: TaskInfo, time: Long): TaskInfo = {
// Task reattempts have a different ID, but the same index as the original.
- new TaskInfo(nextTaskId(), orig.index, orig.attemptNumber + 1, time, orig.executorId,
+ new TaskInfo(
+ nextTaskId(), orig.index, orig.attemptNumber + 1, orig.partitionId, time, orig.executorId,
s"${orig.executorId}.example.com", TaskLocality.PROCESS_LOCAL, orig.speculative)
}
diff --git a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
index 31d8718..3448b4f 100644
--- a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
@@ -140,8 +140,8 @@ class StagePageSuite extends SparkFunSuite with LocalSparkContext {
// Simulate two tasks to test PEAK_EXECUTION_MEMORY correctness
(1 to 2).foreach {
taskId =>
- val taskInfo = new TaskInfo(taskId, taskId, 0, 0, "0", "localhost", TaskLocality.ANY,
- false)
+ val taskInfo = new TaskInfo(taskId, taskId, 0, taskId, 0,
+ "0", "localhost", TaskLocality.ANY, false)
listener.onStageSubmitted(SparkListenerStageSubmitted(stageInfo))
listener.onTaskStart(SparkListenerTaskStart(0, 0, taskInfo))
taskInfo.markFinished(TaskState.FINISHED, System.currentTimeMillis())
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 4eea225..36b61f6 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -45,23 +45,23 @@ class JsonProtocolSuite extends SparkFunSuite {
val stageSubmitted =
SparkListenerStageSubmitted(makeStageInfo(100, 200, 300, 400L, 500L), properties)
val stageCompleted = SparkListenerStageCompleted(makeStageInfo(101, 201, 301, 401L, 501L))
- val taskStart = SparkListenerTaskStart(111, 0, makeTaskInfo(222L, 333, 1, 444L, false))
+ val taskStart = SparkListenerTaskStart(111, 0, makeTaskInfo(222L, 333, 1, 333, 444L, false))
val taskGettingResult =
- SparkListenerTaskGettingResult(makeTaskInfo(1000L, 2000, 5, 3000L, true))
+ SparkListenerTaskGettingResult(makeTaskInfo(1000L, 2000, 5, 2000, 3000L, true))
val taskEnd = SparkListenerTaskEnd(1, 0, "ShuffleMapTask", Success,
- makeTaskInfo(123L, 234, 67, 345L, false),
+ makeTaskInfo(123L, 234, 67, 234, 345L, false),
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
0, 0, 0, 0, 80001L)),
makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, hasHadoopInput = false, hasOutput = false))
val taskEndWithHadoopInput = SparkListenerTaskEnd(1, 0, "ShuffleMapTask", Success,
- makeTaskInfo(123L, 234, 67, 345L, false),
+ makeTaskInfo(123L, 234, 67, 234, 345L, false),
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
0, 0, 0, 0, 80001L)),
makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, hasHadoopInput = true, hasOutput = false))
val taskEndWithOutput = SparkListenerTaskEnd(1, 0, "ResultTask", Success,
- makeTaskInfo(123L, 234, 67, 345L, false),
+ makeTaskInfo(123L, 234, 67, 234, 345L, false),
new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L,
321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L,
0, 0, 0, 0, 80001L)),
@@ -175,7 +175,7 @@ class JsonProtocolSuite extends SparkFunSuite {
val attributes = Map("ContainerId" -> "ct1", "User" -> "spark").toMap
testRDDInfo(makeRddInfo(2, 3, 4, 5L, 6L, DeterministicLevel.DETERMINATE))
testStageInfo(makeStageInfo(10, 20, 30, 40L, 50L))
- testTaskInfo(makeTaskInfo(999L, 888, 55, 777L, false))
+ testTaskInfo(makeTaskInfo(999L, 888, 55, 888, 777L, false))
testTaskMetrics(makeTaskMetrics(
33333L, 44444L, 55555L, 66666L, 7, 8, hasHadoopInput = false, hasOutput = false))
testBlockManagerId(BlockManagerId("Hong", "Kong", 500))
@@ -1021,9 +1021,9 @@ private[spark] object JsonProtocolSuite extends Assertions {
stageInfo
}
- private def makeTaskInfo(a: Long, b: Int, c: Int, d: Long, speculative: Boolean) = {
- val taskInfo = new TaskInfo(a, b, c, d, "executor", "your kind sir", TaskLocality.NODE_LOCAL,
- speculative)
+ private def makeTaskInfo(a: Long, b: Int, c: Int, d: Int, e: Long, speculative: Boolean) = {
+ val taskInfo = new TaskInfo(a, b, c, d, e,
+ "executor", "your kind sir", TaskLocality.NODE_LOCAL, speculative)
taskInfo.setAccumulables(
List(makeAccumulableInfo(1), makeAccumulableInfo(2), makeAccumulableInfo(3, internal = true)))
taskInfo
@@ -1226,6 +1226,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
| "Task ID": 222,
| "Index": 333,
| "Attempt": 1,
+ | "Partition ID": 333,
| "Launch Time": 444,
| "Executor ID": "executor",
| "Host": "your kind sir",
@@ -1273,6 +1274,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
| "Task ID": 1000,
| "Index": 2000,
| "Attempt": 5,
+ | "Partition ID": 2000,
| "Launch Time": 3000,
| "Executor ID": "executor",
| "Host": "your kind sir",
@@ -1326,6 +1328,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
| "Task ID": 123,
| "Index": 234,
| "Attempt": 67,
+ | "Partition ID": 234,
| "Launch Time": 345,
| "Executor ID": "executor",
| "Host": "your kind sir",
@@ -1451,6 +1454,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
| "Task ID": 123,
| "Index": 234,
| "Attempt": 67,
+ | "Partition ID": 234,
| "Launch Time": 345,
| "Executor ID": "executor",
| "Host": "your kind sir",
@@ -1576,6 +1580,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
| "Task ID": 123,
| "Index": 234,
| "Attempt": 67,
+ | "Partition ID": 234,
| "Launch Time": 345,
| "Executor ID": "executor",
| "Host": "your kind sir",
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index 7fdc383..464d5b2 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -96,6 +96,7 @@ local-1426633911242
local-1430917381534
local-1430917381535_1
local-1430917381535_2
+local-1642039451826
DESCRIPTION
NAMESPACE
test_support/*
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index cc148d9..94f4df4 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -56,6 +56,9 @@ object MimaExcludes {
// [SPARK-38432][SQL] Reactor framework so as JDBC dialect could compile filter by self way
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.sources.Filter.toV2"),
+ // [SPARK-37831][CORE] Add task partition id in TaskInfo and Task Metrics
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.TaskData.this"),
+
// [SPARK-37600][BUILD] Upgrade to Hadoop 3.3.2
ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4Compressor"),
ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4Factory"),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/MetricsAggregationBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/MetricsAggregationBenchmark.scala
index aa3988a..5d9bb8a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/MetricsAggregationBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/MetricsAggregationBenchmark.scala
@@ -110,6 +110,7 @@ object MetricsAggregationBenchmark extends BenchmarkBase {
taskId = taskOffset + i.toLong,
index = i,
attemptNumber = 0,
+ partitionId = i,
// The following fields are not used.
launchTime = 0,
executorId = "",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
index 9b5b532..3dabaad 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
@@ -111,6 +111,7 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
taskId = taskId,
attemptNumber = attemptNumber,
index = taskId.toInt,
+ partitionId = taskId.toInt,
// The following fields are not used in tests
launchTime = 0,
executorId = "",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org