You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sa...@apache.org on 2020/12/18 05:26:18 UTC
[spark] branch master updated: [SPARK-26341][WEBUI] Expose executor
memory metrics at the stage level, in the Stages tab
This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 25c6cc2 [SPARK-26341][WEBUI] Expose executor memory metrics at the stage level, in the Stages tab
25c6cc2 is described below
commit 25c6cc25f74e8a24aa424f6596a574f26ae80e1d
Author: angerszhu <an...@gmail.com>
AuthorDate: Fri Dec 18 14:24:53 2020 +0900
[SPARK-26341][WEBUI] Expose executor memory metrics at the stage level, in the Stages tab
### What changes were proposed in this pull request?
Expose executor memory metrics at the stage level, in the Stages tab,
Current like below, and I am not sure which column we will truly need.
### Why are the changes needed?
User can know executor jvm usage more directly in SparkUI
### Does this PR introduce any user-facing change?
User can know executor jvm usage more directly in SparkUI
### How was this patch tested?
Manual Tested
Closes #30573 from AngersZhuuuu/SPARK-26341.
Authored-by: angerszhu <an...@gmail.com>
Signed-off-by: Kousuke Saruta <sa...@oss.nttdata.com>
---
.../org/apache/spark/ui/static/stagepage.js | 163 +++++++++++++++++----
.../spark/ui/static/stagespage-template.html | 4 +
.../apache/spark/status/AppStatusListener.scala | 3 +
.../excludeOnFailure_for_stage_expectation.json | 44 ++++++
...xcludeOnFailure_node_for_stage_expectation.json | 110 ++++++++++++++
.../one_stage_attempt_json_expectation.json | 22 +++
.../one_stage_json_expectation.json | 22 +++
.../stage_with_accumulable_json_expectation.json | 22 +++
8 files changed, 365 insertions(+), 25 deletions(-)
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
index 2877aa8..336edff 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
@@ -243,23 +243,39 @@ function createRowMetadataForColumn(colKey, data, checkboxId) {
}
function reselectCheckboxesBasedOnTaskTableState() {
- var allChecked = true;
+ var taskSummaryHasSelected = false;
+ var executorSummaryHasSelected = false;
+ var allTaskSummaryChecked = true;
+ var allExecutorSummaryChecked = true;
var taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
if (typeof taskTableSelector !== 'undefined' && taskSummaryMetricsTableCurrentStateArray.length > 0) {
for (var k = 0; k < optionalColumns.length; k++) {
if (taskTableSelector.column(optionalColumns[k]).visible()) {
+ taskSummaryHasSelected = true;
$("#box-"+optionalColumns[k]).prop('checked', true);
taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]);
taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
} else {
- allChecked = false;
+ allTaskSummaryChecked = false;
}
}
- if (allChecked) {
- $("#box-0").prop('checked', true);
- }
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray);
}
+
+ if (typeof executorSummaryTableSelector !== 'undefined') {
+ for (var k = 0; k < executorOptionalColumns.length; k++) {
+ if (executorSummaryTableSelector.column(executorOptionalColumns[k]).visible()) {
+ executorSummaryHasSelected = true;
+ $("#executor-box-"+executorOptionalColumns[k]).prop('checked', true);
+ } else {
+ allExecutorSummaryChecked = false;
+ }
+ }
+ }
+
+ if ((taskSummaryHasSelected || executorSummaryHasSelected) && allTaskSummaryChecked && allExecutorSummaryChecked) {
+ $("#box-0").prop('checked', true);
+ }
}
function getStageAttemptId() {
@@ -278,6 +294,9 @@ var taskSummaryMetricsDataTable;
var optionalColumns = [11, 12, 13, 14, 15, 16, 17, 21];
var taskTableSelector;
+var executorOptionalColumns = [15, 16, 17, 18];
+var executorSummaryTableSelector;
+
$(document).ready(function () {
setDataTableDefaults();
@@ -288,14 +307,18 @@ $(document).ready(function () {
"</a></div>" +
"<div class='container-fluid-div ml-4 d-none' id='toggle-metrics'>" +
"<div id='select_all' class='select-all-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-0' data-column='0'> Select All</div>" +
- "<div id='scheduler_delay' class='scheduler-delay-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-11' data-column='11'> Scheduler Delay</div>" +
- "<div id='task_deserialization_time' class='task-deserialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-12' data-column='12'> Task Deserialization Time</div>" +
- "<div id='shuffle_read_blocked_time' class='shuffle-read-blocked-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-13' data-column='13'> Shuffle Read Blocked Time</div>" +
- "<div id='shuffle_remote_reads' class='shuffle-remote-reads-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-14' data-column='14'> Shuffle Remote Reads</div>" +
- "<div id='shuffle_write_time' class='shuffle-write-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-21' data-column='21'> Shuffle Write Time</div>" +
- "<div id='result_serialization_time' class='result-serialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-15' data-column='15'> Result Serialization Time</div>" +
- "<div id='getting_result_time' class='getting-result-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-16' data-column='16'> Getting Result Time</div>" +
- "<div id='peak_execution_memory' class='peak-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-17' data-column='17'> Peak Execution Memory</div>" +
+ "<div id='scheduler_delay' class='scheduler-delay-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-11' data-column='11' data-metrics-type='task'> Scheduler Delay</div>" +
+ "<div id='task_deserialization_time' class='task-deserialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-12' data-column='12' data-metrics-type='task'> Task Deserialization Time</div>" +
+ "<div id='shuffle_read_blocked_time' class='shuffle-read-blocked-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-13' data-column='13' data-metrics-type='task'> Shuffle Read Blocked Time</div>" +
+ "<div id='shuffle_remote_reads' class='shuffle-remote-reads-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-14' data-column='14' data-metrics-type='task'> Shuffle Remote Reads</div>" +
+ "<div id='shuffle_write_time' class='shuffle-write-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-21' data-column='21' data-metrics-type='task'> Shuffle Write Time</div>" +
+ "<div id='result_serialization_time' class='result-serialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-15' data-column='15' data-metrics-type='task'> Result Serialization Time</div>" +
+ "<div id='getting_result_time' class='getting-result-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-16' data-column='16' data-metrics-type='task'> Getting Result Time</div>" +
+ "<div id='peak_execution_memory' class='peak-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-17' data-column='17' data-metrics-type='task'> Peak Execution Memory</div>" +
+ "<div id='executor_jvm_on_off_heap_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-15' data-column='15' data-metrics-type='executor'> Peak JVM Memory OnHeap / OffHeap</div>" +
+ "<div id='executor_on_off_heap_execution_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-16' data-column='16' data-metrics-type='executor'> Peak Execution Memory OnHeap / OffHeap</div>" +
+ "<div id='executor_on_off_heap_storage_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-17' data-column='17' data-metrics-type='executor'> Peak Storage Memory OnHeap / OffHeap</div>" +
+ "<div id='executor_direct_mapped_pool_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-18' data-column='18' data-metrics-type='executor'> Peak Pool Memory Direct / Mapped</div>" +
"</div>");
$('#scheduler_delay').attr("data-toggle", "tooltip")
@@ -463,15 +486,95 @@ $(document).ready(function () {
data : function (row, type) {
return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : "";
}
+ },
+ {
+ data : function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics;
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.JVMHeapMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+
+ }
+ },
+ {
+ data : function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.OnHeapExecutionMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+ }
+ },
+ {
+ data : function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.OnHeapStorageMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+ }
+ },
+ {
+ data : function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.DirectPoolMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.MappedPoolMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+ }
}
],
+ "columnDefs": [
+ { "visible": false, "targets": 15 },
+ { "visible": false, "targets": 16 },
+ { "visible": false, "targets": 17 },
+ { "visible": false, "targets": 18 }
+ ],
+ "deferRender": true,
"order": [[0, "asc"]],
"bAutoWidth": false,
"oLanguage": {
"sEmptyTable": "No data to show yet"
}
};
- var executorSummaryTableSelector =
+ executorSummaryTableSelector =
$("#summary-executor-table").DataTable(executorSummaryConf);
$('#parent-container [data-toggle="tooltip"]').tooltip();
@@ -923,30 +1026,40 @@ $(document).ready(function () {
var para = $(this).attr('data-column');
if (para == "0") {
var allColumns = taskTableSelector.columns(optionalColumns);
+ var executorAllColumns = executorSummaryTableSelector.columns(executorOptionalColumns);
if ($(this).is(":checked")) {
$(".toggle-vis").prop('checked', true);
allColumns.visible(true);
+ executorAllColumns.visible(true);
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray);
} else {
$(".toggle-vis").prop('checked', false);
allColumns.visible(false);
+ executorAllColumns.visible(false);
var taskSummaryMetricsTableFilteredArray =
taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
}
} else {
- var column = taskTableSelector.column(para);
- // Toggle the visibility
- column.visible(!column.visible());
- var taskSummaryMetricsTableFilteredArray = [];
- if ($(this).is(":checked")) {
- taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]);
- taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
- } else {
- taskSummaryMetricsTableFilteredArray =
- taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para);
+ var dataMetricsType = $(this).attr("data-metrics-type");
+ if (dataMetricsType === 'task') {
+ var column = taskTableSelector.column(para);
+ // Toggle the visibility
+ column.visible(!column.visible());
+ var taskSummaryMetricsTableFilteredArray = [];
+ if ($(this).is(":checked")) {
+ taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]);
+ taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
+ } else {
+ taskSummaryMetricsTableFilteredArray =
+ taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para);
+ }
+ createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
+ }
+ if (dataMetricsType === "executor") {
+ var column = executorSummaryTableSelector.column(para);
+ column.visible(!column.visible());
}
- createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
}
});
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html b/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
index 9b40d0d..b938158 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
@@ -59,6 +59,10 @@ limitations under the License.
<th><span id="executor-summary-shuffle-write">Shuffle Write Size / Records</span></th>
<th>Spill (Memory) </th>
<th>Spill (Disk) </th>
+ <th>Peak JVM Memory OnHeap / OffHeap</th>
+ <th>Peak Execution Memory OnHeap / OffHeap</th>
+ <th>Peak Storage Memory OnHeap / OffHeap</th>
+ <th>Peak Pool Memory Direct / Mapped</th>
</tr>
</thead>
<tbody>
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index 5b0c1dc..0722095 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -687,6 +687,9 @@ private[spark] class AppStatusListener(
stage.killedSummary = killedTasksSummary(event.reason, stage.killedSummary)
}
stage.activeTasksPerExecutor(event.taskInfo.executorId) -= 1
+
+ stage.executorSummary(event.taskInfo.executorId).peakExecutorMetrics
+ .compareAndUpdatePeakValues(event.taskExecutorMetrics)
// [SPARK-24415] Wait for all tasks to finish before removing stage from live list
val removeStage =
stage.activeTasks == 0 &&
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
index a69940f..ab9a8b7 100644
--- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
@@ -698,6 +698,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : true,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : true
},
"1" : {
@@ -716,6 +738,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : false,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : false
}
},
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
index bda9cae..1c569c1 100644
--- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
@@ -806,6 +806,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : true,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : true
},
"5" : {
@@ -824,6 +846,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : true,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : true
},
"1" : {
@@ -842,6 +886,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : false,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : false
},
"2" : {
@@ -860,6 +926,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : false,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : false
},
"3" : {
@@ -878,6 +966,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : true,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : true
}
},
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index 41e54c6..b1eab0d 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -460,6 +460,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : false,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : false
}
},
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 7a6685a..6dfdd27 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -460,6 +460,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : false,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : false
}
},
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index 066b6a4..a2cfd9d 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -504,6 +504,28 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"isBlacklistedForStage" : false,
+ "peakMemoryMetrics" : {
+ "JVMHeapMemory" : 0,
+ "JVMOffHeapMemory" : 0,
+ "OnHeapExecutionMemory" : 0,
+ "OffHeapExecutionMemory" : 0,
+ "OnHeapStorageMemory" : 0,
+ "OffHeapStorageMemory" : 0,
+ "OnHeapUnifiedMemory" : 0,
+ "OffHeapUnifiedMemory" : 0,
+ "DirectPoolMemory" : 0,
+ "MappedPoolMemory" : 0,
+ "ProcessTreeJVMVMemory" : 0,
+ "ProcessTreeJVMRSSMemory" : 0,
+ "ProcessTreePythonVMemory" : 0,
+ "ProcessTreePythonRSSMemory" : 0,
+ "ProcessTreeOtherVMemory" : 0,
+ "ProcessTreeOtherRSSMemory" : 0,
+ "MinorGCCount" : 0,
+ "MinorGCTime" : 0,
+ "MajorGCCount" : 0,
+ "MajorGCTime" : 0
+ },
"isExcludedForStage" : false
}
},
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org