You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2024/08/07 02:52:22 UTC
(spark) branch master updated: [SPARK-49126][CORE] Move `spark.history.ui.maxApplications` config definition to `History.scala`
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 514f272273d7 [SPARK-49126][CORE] Move `spark.history.ui.maxApplications` config definition to `History.scala`
514f272273d7 is described below
commit 514f272273d79c067a7d4ccbba3b4c8aaef69d27
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Wed Aug 7 11:52:10 2024 +0900
[SPARK-49126][CORE] Move `spark.history.ui.maxApplications` config definition to `History.scala`
### What changes were proposed in this pull request?
This PR aims to move `spark.history.ui.maxApplications` config definition to `History.scala` in order to collect all `spark.history.ui.*` into one file.
### Why are the changes needed?
After this PR, we have all `spark.history.ui.*` configuration definition in `History.scala` consistently.
```
$ git grep spark.history.ui
core/src/main/scala/org/apache/spark/internal/config/History.scala: val HISTORY_SERVER_UI_PORT = ConfigBuilder("spark.history.ui.port")
core/src/main/scala/org/apache/spark/internal/config/History.scala: val HISTORY_SERVER_UI_ACLS_ENABLE = ConfigBuilder("spark.history.ui.acls.enable")
core/src/main/scala/org/apache/spark/internal/config/History.scala: val HISTORY_SERVER_UI_ADMIN_ACLS = ConfigBuilder("spark.history.ui.admin.acls")
core/src/main/scala/org/apache/spark/internal/config/History.scala: val HISTORY_SERVER_UI_ADMIN_ACLS_GROUPS = ConfigBuilder("spark.history.ui.admin.acls.groups")
core/src/main/scala/org/apache/spark/internal/config/History.scala: val HISTORY_UI_MAX_APPS = ConfigBuilder("spark.history.ui.maxApplications")
docs/monitoring.md: <td>spark.history.ui.maxApplications</td>
docs/monitoring.md: <td>spark.history.ui.port</td>
docs/security.md: <td><code>spark.history.ui.acls.enable</code></td>
docs/security.md: <td><code>spark.history.ui.admin.acls</code></td>
docs/security.md: <td><code>spark.history.ui.admin.acls.groups</code></td>
docs/security.md: <td><code>spark.history.ui.port</code></td>
resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ApplicationMasterSuite.scala: "http://${hadoopconf-yarn.resourcemanager.hostname}:${spark.history.ui.port}")
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #47636 from dongjoon-hyun/SPARK-49126.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../scala/org/apache/spark/deploy/history/HistoryServer.scala | 3 +--
.../src/main/scala/org/apache/spark/internal/config/History.scala | 8 ++++++++
.../src/main/scala/org/apache/spark/internal/config/package.scala | 7 -------
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index 6e559dc4492e..37968e14fb03 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -30,7 +30,6 @@ import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.Utils.addRenderLogHandler
import org.apache.spark.internal.{Logging, MDC}
import org.apache.spark.internal.LogKeys._
-import org.apache.spark.internal.config._
import org.apache.spark.internal.config.History
import org.apache.spark.internal.config.UI._
import org.apache.spark.status.api.v1.{ApiRootResource, ApplicationInfo, UIRoot}
@@ -65,7 +64,7 @@ class HistoryServer(
private val retainedApplications = conf.get(History.RETAINED_APPLICATIONS)
// How many applications the summary ui displays
- private[history] val maxApplications = conf.get(HISTORY_UI_MAX_APPS);
+ private[history] val maxApplications = conf.get(History.HISTORY_UI_MAX_APPS);
// application
private val appCache = new ApplicationCache(this, retainedApplications, new SystemClock())
diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index 64a8681ca295..b7fab8d1e8f4 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -201,6 +201,14 @@ private[spark] object History {
.toSequence
.createWithDefault(Nil)
+ val HISTORY_UI_MAX_APPS = ConfigBuilder("spark.history.ui.maxApplications")
+ .version("2.0.1")
+ .doc("The number of applications to display on the history summary page. Application UIs " +
+ "are still available by accessing their URLs directly even if they are not displayed on " +
+ "the history summary page.")
+ .intConf
+ .createWithDefault(Integer.MAX_VALUE)
+
val NUM_REPLAY_THREADS = ConfigBuilder("spark.history.fs.numReplayThreads")
.version("2.0.0")
.doc("Number of threads that will be used by history server to process event logs.")
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index c40645065bb5..940d72df5df6 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1110,13 +1110,6 @@ package object config {
.stringConf
.createOptional
- // To limit how many applications are shown in the History Server summary ui
- private[spark] val HISTORY_UI_MAX_APPS =
- ConfigBuilder("spark.history.ui.maxApplications")
- .version("2.0.1")
- .intConf
- .createWithDefault(Integer.MAX_VALUE)
-
private[spark] val IO_ENCRYPTION_ENABLED = ConfigBuilder("spark.io.encryption.enabled")
.version("2.1.0")
.booleanConf
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org