You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2023/03/09 05:34:00 UTC
[spark] branch master updated: [SPARK-42697][WEBUI] Fix /api/v1/applications to return total uptime instead of 0 for the duration field
This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d3d8fdc2882 [SPARK-42697][WEBUI] Fix /api/v1/applications to return total uptime instead of 0 for the duration field
d3d8fdc2882 is described below
commit d3d8fdc2882f5c084897ca9b2af9a063358f3a21
Author: Kent Yao <ya...@apache.org>
AuthorDate: Thu Mar 9 13:33:43 2023 +0800
[SPARK-42697][WEBUI] Fix /api/v1/applications to return total uptime instead of 0 for the duration field
### What changes were proposed in this pull request?
Fix /api/v1/applications to return total uptime instead of 0 for duration
### Why are the changes needed?
Fix REST API OneApplicationResource
### Does this PR introduce _any_ user-facing change?
yes, /api/v1/applications will return the total uptime instead of 0 for the duration
### How was this patch tested?
locally build and run
```json
[ {
"id" : "local-1678183638394",
"name" : "SparkSQL::10.221.102.180",
"attempts" : [ {
"startTime" : "2023-03-07T10:07:17.754GMT",
"endTime" : "1969-12-31T23:59:59.999GMT",
"lastUpdated" : "2023-03-07T10:07:17.754GMT",
"duration" : 20317,
"sparkUser" : "kentyao",
"completed" : false,
"appSparkVersion" : "3.5.0-SNAPSHOT",
"startTimeEpoch" : 1678183637754,
"endTimeEpoch" : -1,
"lastUpdatedEpoch" : 1678183637754
} ]
} ]
```
Closes #40313 from yaooqinn/SPARK-42697.
Authored-by: Kent Yao <ya...@apache.org>
Signed-off-by: Kent Yao <ya...@apache.org>
---
core/src/main/scala/org/apache/spark/ui/SparkUI.scala | 2 +-
core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala | 9 ++++++++-
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index db1f8bc1a2f..ac154b79385 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -167,7 +167,7 @@ private[spark] class SparkUI private (
attemptId = None,
startTime = new Date(startTime),
endTime = new Date(-1),
- duration = 0,
+ duration = System.currentTimeMillis() - startTime,
lastUpdated = new Date(startTime),
sparkUser = getSparkUser,
completed = false,
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 45348b2e9a7..79496bba667 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -700,7 +700,14 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers {
parseDate(attempts(0) \ "startTime") should be (sc.startTime)
parseDate(attempts(0) \ "endTime") should be (-1)
val oneAppJsonAst = getJson(sc.ui.get, "")
- oneAppJsonAst should be (appListJsonAst.children(0))
+ val duration = attempts(0) \ "duration"
+ oneAppJsonAst \\ "duration" should not be duration
+ // SPARK-42697: duration will increase as the app is running
+ // Replace the duration before we compare the full JObjects
+ val durationAdjusted = oneAppJsonAst.transformField {
+ case ("duration", _) => ("duration", duration)
+ }
+ durationAdjusted should be (appListJsonAst.children(0))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org