You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sa...@apache.org on 2020/06/15 01:44:16 UTC
[spark] branch branch-3.0 updated: [SPARK-31983][WEBUI][3.0] Fix
sorting for duration column in structured streaming tab
This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 9ca5934 [SPARK-31983][WEBUI][3.0] Fix sorting for duration column in structured streaming tab
9ca5934 is described below
commit 9ca5934cb6bf8257d91a33fcf6f2738822fae34a
Author: iRakson <ra...@gmail.com>
AuthorDate: Mon Jun 15 10:39:55 2020 +0900
[SPARK-31983][WEBUI][3.0] Fix sorting for duration column in structured streaming tab
### What changes were proposed in this pull request?
Sorting result for duration column in tables of structured streaming tab is wrong sometimes.
<img width="1677" alt="Screenshot 2020-06-13 at 1 58 53 PM" src="https://user-images.githubusercontent.com/15366835/84572178-10755700-adb6-11ea-9131-338e8ba7fb24.png">
We are sorting on string, which results in this behaviour.
`sorttable_numeric` and `sorttable_customkey` is used to fix this.
Refer [this](https://github.com/apache/spark/pull/28752#issuecomment-643451586) and [this](https://github.com/apache/spark/pull/28752#issuecomment-643569254)
After changes :
<img width="1677" alt="Screenshot 2020-06-13 at 8 05 32 PM" src="https://user-images.githubusercontent.com/15366835/84572299-a8734080-adb6-11ea-9aa3-b4bc594de4cf.png">
### Why are the changes needed?
Sorting results are wrong for duration column in tables of structured streaming tab.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Screenshots attached.
Closes #28823 from iRakson/testsort.
Authored-by: iRakson <ra...@gmail.com>
Signed-off-by: Kousuke Saruta <sa...@oss.nttdata.com>
---
.../sql/streaming/ui/StreamingQueryPage.scala | 23 ++++++++++++++++------
1 file changed, 17 insertions(+), 6 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
index 7336765..43b93a3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
@@ -57,12 +57,12 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
val name = UIUtils.getQueryName(query)
val status = UIUtils.getQueryStatus(query)
val duration = if (queryActive) {
- SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.startTimestamp)
+ System.currentTimeMillis() - query.startTimestamp
} else {
withNoProgress(query, {
val endTimeMs = query.lastProgress.timestamp
- SparkUIUtils.formatDurationVerbose(parseProgressTimestamp(endTimeMs) - query.startTimestamp)
- }, "-")
+ parseProgressTimestamp(endTimeMs) - query.startTimestamp
+ }, 0)
}
<tr>
@@ -71,7 +71,9 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
<td> {query.id} </td>
<td> <a href={statisticsLink}> {query.runId} </a> </td>
<td> {SparkUIUtils.formatDate(query.startTimestamp)} </td>
- <td> {duration} </td>
+ <td sorttable_customkey={duration.toString}>
+ {SparkUIUtils.formatDurationVerbose(duration)}
+ </td>
<td> {withNoProgress(query, {
(query.recentProgress.map(p => withNumberInvalid(p.inputRowsPerSecond)).sum /
query.recentProgress.length).formatted("%.2f") }, "NaN")}
@@ -93,8 +95,13 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
"Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
"Avg Process /sec", "Lastest Batch")
+ val headerCss = Seq("", "", "", "", "", "sorttable_numeric", "sorttable_numeric",
+ "sorttable_numeric", "")
+ // header classes size must be equal to header row size
+ assert(headerRow.size == headerCss.size)
+
Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = true),
- activeQueries, true, Some("activeQueries-table"), Seq(null), false))
+ activeQueries, true, Some("activeQueries-table"), headerCss, false))
} else {
None
}
@@ -104,8 +111,12 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
"Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
"Avg Process /sec", "Lastest Batch", "Error")
+ val headerCss = Seq("", "", "", "", "", "sorttable_numeric", "sorttable_numeric",
+ "sorttable_numeric", "", "")
+ assert(headerRow.size == headerCss.size)
+
Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = false),
- inactiveQueries, true, Some("completedQueries-table"), Seq(null), false))
+ inactiveQueries, true, Some("completedQueries-table"), headerCss, false))
} else {
None
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org