You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/08/08 05:22:32 UTC
spark git commit: [SPARK-16919] Configurable update interval for
console progress bar
Repository: spark
Updated Branches:
refs/heads/master a16983c97 -> e076fb05a
[SPARK-16919] Configurable update interval for console progress bar
## What changes were proposed in this pull request?
Currently the update interval for the console progress bar is hardcoded. This PR makes it configurable for users.
## How was this patch tested?
Ran a long running job and with a high value of update interval, the updates were shown less frequently.
Author: Tejas Patil <te...@fb.com>
Closes #14507 from tejasapatil/SPARK-16919.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e076fb05
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e076fb05
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e076fb05
Branch: refs/heads/master
Commit: e076fb05ac83a3ed6995e29bb03ea07ea05e39db
Parents: a16983c
Author: Tejas Patil <te...@fb.com>
Authored: Mon Aug 8 06:22:37 2016 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Mon Aug 8 06:22:37 2016 +0100
----------------------------------------------------------------------
.../apache/spark/ui/ConsoleProgressBar.scala | 23 ++++++++++----------
1 file changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e076fb05/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala b/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
index 2719e1e..3ae80ec 100644
--- a/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
+++ b/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
@@ -30,22 +30,23 @@ import org.apache.spark.internal.Logging
*/
private[spark] class ConsoleProgressBar(sc: SparkContext) extends Logging {
// Carriage return
- val CR = '\r'
+ private val CR = '\r'
// Update period of progress bar, in milliseconds
- val UPDATE_PERIOD = 200L
+ private val updatePeriodMSec =
+ sc.getConf.getTimeAsMs("spark.ui.consoleProgress.update.interval", "200")
// Delay to show up a progress bar, in milliseconds
- val FIRST_DELAY = 500L
+ private val firstDelayMSec = 500L
// The width of terminal
- val TerminalWidth = if (!sys.env.getOrElse("COLUMNS", "").isEmpty) {
+ private val TerminalWidth = if (!sys.env.getOrElse("COLUMNS", "").isEmpty) {
sys.env.get("COLUMNS").get.toInt
} else {
80
}
- var lastFinishTime = 0L
- var lastUpdateTime = 0L
- var lastProgressBar = ""
+ private var lastFinishTime = 0L
+ private var lastUpdateTime = 0L
+ private var lastProgressBar = ""
// Schedule a refresh thread to run periodically
private val timer = new Timer("refresh progress", true)
@@ -53,19 +54,19 @@ private[spark] class ConsoleProgressBar(sc: SparkContext) extends Logging {
override def run() {
refresh()
}
- }, FIRST_DELAY, UPDATE_PERIOD)
+ }, firstDelayMSec, updatePeriodMSec)
/**
* Try to refresh the progress bar in every cycle
*/
private def refresh(): Unit = synchronized {
val now = System.currentTimeMillis()
- if (now - lastFinishTime < FIRST_DELAY) {
+ if (now - lastFinishTime < firstDelayMSec) {
return
}
val stageIds = sc.statusTracker.getActiveStageIds()
val stages = stageIds.flatMap(sc.statusTracker.getStageInfo).filter(_.numTasks() > 1)
- .filter(now - _.submissionTime() > FIRST_DELAY).sortBy(_.stageId())
+ .filter(now - _.submissionTime() > firstDelayMSec).sortBy(_.stageId())
if (stages.length > 0) {
show(now, stages.take(3)) // display at most 3 stages in same time
}
@@ -94,7 +95,7 @@ private[spark] class ConsoleProgressBar(sc: SparkContext) extends Logging {
header + bar + tailer
}.mkString("")
- // only refresh if it's changed of after 1 minute (or the ssh connection will be closed
+ // only refresh if it's changed OR after 1 minute (or the ssh connection will be closed
// after idle some time)
if (bar != lastProgressBar || now - lastUpdateTime > 60 * 1000L) {
System.err.print(CR + bar)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org