You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2020/04/07 05:50:30 UTC
[spark] 02/03: [SPARK-31109][MESOS][DOC] Add version information to
the configuration of Mesos
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
commit d8b86fd5fb491c3e188fb9b30ca1f4b6a8384188
Author: beliefer <be...@163.com>
AuthorDate: Thu Mar 12 11:02:29 2020 +0900
[SPARK-31109][MESOS][DOC] Add version information to the configuration of Mesos
### What changes were proposed in this pull request?
Add version information to the configuration of `Mesos`.
I sorted out some information show below.
Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.mesos.$taskType.secret.names | 2.3.0 | SPARK-22131 | 5415963d2caaf95604211419ffc4e29fff38e1d7#diff-91e6e5f871160782dc50d4060d6faea3 |
spark.mesos.$taskType.secret.values | 2.3.0 | SPARK-22131 | 5415963d2caaf95604211419ffc4e29fff38e1d7#diff-91e6e5f871160782dc50d4060d6faea3 |
spark.mesos.$taskType.secret.envkeys | 2.3.0 | SPARK-22131 | 5415963d2caaf95604211419ffc4e29fff38e1d7#diff-91e6e5f871160782dc50d4060d6faea3 |
spark.mesos.$taskType.secret.filenames | 2.3.0 | SPARK-22131 | 5415963d2caaf95604211419ffc4e29fff38e1d7#diff-91e6e5f871160782dc50d4060d6faea3 |
spark.mesos.principal | 1.5.0 | SPARK-6284 | d86bbb4e286f16f77ba125452b07827684eafeed#diff-02a6d899f7a529eb7cfbb12182a110b0 |
spark.mesos.principal.file | 2.4.0 | SPARK-16501 | 7f10cf83f311526737fc96d5bb8281d12e41932f#diff-daf48dabbe58afaeed8787751750b01d |
spark.mesos.secret | 1.5.0 | SPARK-6284 | d86bbb4e286f16f77ba125452b07827684eafeed#diff-02a6d899f7a529eb7cfbb12182a110b0 |
spark.mesos.secret.file | 2.4.0 | SPARK-16501 | 7f10cf83f311526737fc96d5bb8281d12e41932f#diff-daf48dabbe58afaeed8787751750b01d |
spark.shuffle.cleaner.interval | 2.0.0 | SPARK-12583 | 310981d49a332bd329303f610b150bbe02cf5f87#diff-2fafefee94f2a2023ea9765536870258 |
spark.mesos.dispatcher.webui.url | 2.0.0 | SPARK-13492 | a4a0addccffb7cd0ece7947d55ce2538afa54c97#diff-f541460c7a74cee87cbb460b3b01665e |
spark.mesos.dispatcher.historyServer.url | 2.1.0 | SPARK-16809 | 62e62124419f3fa07b324f5e42feb2c5b4fde715#diff-3779e2035d9a09fa5f6af903925b9512 |
spark.mesos.driver.labels | 2.3.0 | SPARK-21000 | 8da3f7041aafa71d7596b531625edb899970fec2#diff-91e6e5f871160782dc50d4060d6faea3 |
spark.mesos.driver.webui.url | 2.0.0 | SPARK-13492 | a4a0addccffb7cd0ece7947d55ce2538afa54c97#diff-e3a5e67b8de2069ce99801372e214b8e |
spark.mesos.driver.failoverTimeout | 2.3.0 | SPARK-21456 | c42ef953343073a50ef04c5ce848b574ff7f2238#diff-91e6e5f871160782dc50d4060d6faea3 |
spark.mesos.network.name | 2.1.0 | SPARK-18232 | d89bfc92302424406847ac7a9cfca714e6b742fc#diff-ab5bf34f1951a8f7ea83c9456a6c3ab7 |
spark.mesos.network.labels | 2.3.0 | SPARK-21694 | ce0d3bb377766bdf4df7852272557ae846408877#diff-91e6e5f871160782dc50d4060d6faea3 |
spark.mesos.driver.constraints | 2.2.1 | SPARK-19606 | f6ee3d90d5c299e67ae6e2d553c16c0d9759d4b5#diff-91e6e5f871160782dc50d4060d6faea3 |
spark.mesos.driver.frameworkId | 2.1.0 | SPARK-16809 | 62e62124419f3fa07b324f5e42feb2c5b4fde715#diff-02a6d899f7a529eb7cfbb12182a110b0 |
spark.executor.uri | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-a885e7df97790e9b59c21c63353e7476 |
spark.mesos.proxy.baseURL | 2.3.0 | SPARK-13041 | 663f30d14a0c9219e07697af1ab56e11a714d9a6#diff-0b9b4e122eb666155aa189a4321a6ca8 |
spark.mesos.coarse | 0.6.0 | None | 63051dd2bcc4bf09d413ff7cf89a37967edc33ba#diff-eaf125f56ce786d64dcef99cf446a751 |
spark.mesos.coarse.shutdownTimeout | 2.0.0 | SPARK-12330 | c756bda477f458ba4aad7fdb2026263507e0ad9b#diff-d425d35aa23c47a62fbb538554f2f2cf |
spark.mesos.maxDrivers | 1.4.0 | SPARK-5338 | 53befacced828bbac53c6e3a4976ec3f036bae9e#diff-b964c449b99c51f0a5fd77270b2951a4 |
spark.mesos.retainedDrivers | 1.4.0 | SPARK-5338 | 53befacced828bbac53c6e3a4976ec3f036bae9e#diff-b964c449b99c51f0a5fd77270b2951a4 |
spark.mesos.cluster.retry.wait.max | 1.4.0 | SPARK-5338 | 53befacced828bbac53c6e3a4976ec3f036bae9e#diff-b964c449b99c51f0a5fd77270b2951a4 |
spark.mesos.fetcherCache.enable | 2.1.0 | SPARK-15994 | e34b4e12673fb76c92f661d7c03527410857a0f8#diff-772ea7311566edb25f11a4c4f882179a |
spark.mesos.appJar.local.resolution.mode | 2.4.0 | SPARK-24326 | 22df953f6bb191858053eafbabaa5b3ebca29f56#diff-6e4d0a0445975f03f975fdc1e3d80e49 |
spark.mesos.rejectOfferDuration | 2.2.0 | SPARK-19702 | 2e30c0b9bcaa6f7757bd85d1f1ec392d5f916f83#diff-daf48dabbe58afaeed8787751750b01d |
spark.mesos.rejectOfferDurationForUnmetConstraints | 1.6.0 | SPARK-10471 | 74f50275e429e649212928a9f36552941b862edc#diff-02a6d899f7a529eb7cfbb12182a110b0 |
spark.mesos.rejectOfferDurationForReachedMaxCores | 2.0.0 | SPARK-13001 | 1e7d9bfb5a41f5c2479ab3b4d4081f00bf00bd31#diff-02a6d899f7a529eb7cfbb12182a110b0 |
spark.mesos.uris | 1.5.0 | SPARK-8798 | a2f805729b401c68b60bd690ad02533b8db57b58#diff-e3a5e67b8de2069ce99801372e214b8e |
spark.mesos.executor.home | 1.1.1 | SPARK-3264 | 069ecfef02c4af69fc0d3755bd78be321b68b01d#diff-e3a5e67b8de2069ce99801372e214b8e |
spark.mesos.mesosExecutor.cores | 1.4.0 | SPARK-6350 | 6fbeb82e13db7117d8f216e6148632490a4bc5be#diff-e3a5e67b8de2069ce99801372e214b8e |
spark.mesos.extra.cores | 0.6.0 | None | 2d761e3353651049f6707c74bb5ffdd6e86f6f35#diff-37af8c6e3634f97410ade813a5172621 |
spark.mesos.executor.memoryOverhead | 1.1.1 | SPARK-3535 | 6f150978477830bbc14ba983786dd2bce12d1fe2#diff-6b498f5407d10e848acac4a1b182457c |
spark.mesos.executor.docker.image | 1.4.0 | SPARK-2691 | 8f50a07d2188ccc5315d979755188b1e5d5b5471#diff-e3a5e67b8de2069ce99801372e214b8e |
spark.mesos.executor.docker.forcePullImage | 2.1.0 | SPARK-15271 | 978cd5f125eb5a410bad2e60bf8385b11cf1b978#diff-0dd025320c7ecda2ea310ed7172d7f5a |
spark.mesos.executor.docker.portmaps | 1.4.0 | SPARK-7373 | 226033cfffa2f37ebaf8bc2c653f094e91ef0c9b#diff-b964c449b99c51f0a5fd77270b2951a4 |
spark.mesos.executor.docker.parameters | 2.2.0 | SPARK-19740 | a888fed3099e84c2cf45e9419f684a3658ada19d#diff-4139e6605a8c7f242f65cde538770c99 |
spark.mesos.executor.docker.volumes | 1.4.0 | SPARK-7373 | 226033cfffa2f37ebaf8bc2c653f094e91ef0c9b#diff-b964c449b99c51f0a5fd77270b2951a4 |
spark.mesos.gpus.max | 2.1.0 | SPARK-14082 | 29f186bfdf929b1e8ffd8e33ee37b76d5dc5af53#diff-d427ee890b913c5a7056be21eb4f39d7 |
spark.mesos.task.labels | 2.2.0 | SPARK-20085 | c8fc1f3badf61bcfc4bd8eeeb61f73078ca068d1#diff-387c5d0c916278495fc28420571adf9e |
spark.mesos.constraints | 1.5.0 | SPARK-6707 | 1165b17d24cdf1dbebb2faca14308dfe5c2a652c#diff-e3a5e67b8de2069ce99801372e214b8e |
spark.mesos.containerizer | 2.1.0 | SPARK-16637 | 266b92faffb66af24d8ed2725beb80770a2d91f8#diff-0dd025320c7ecda2ea310ed7172d7f5a |
spark.mesos.role | 1.5.0 | SPARK-6284 | d86bbb4e286f16f77ba125452b07827684eafeed#diff-02a6d899f7a529eb7cfbb12182a110b0 |
The following appears in the document | | | |
spark.mesos.driverEnv.[EnvironmentVariableName] | 2.1.0 | SPARK-16194 | 235cb256d06653bcde4c3ed6b081503a94996321#diff-b964c449b99c51f0a5fd77270b2951a4 |
spark.mesos.dispatcher.driverDefault.[PropertyName] | 2.1.0 | SPARK-16927 and SPARK-16923 | eca58755fbbc11937b335ad953a3caff89b818e6#diff-b964c449b99c51f0a5fd77270b2951a4 |
### Why are the changes needed?
Supplemental configuration version information.
### Does this PR introduce any user-facing change?
'No'.
### How was this patch tested?
Exists UT
Closes #27863 from beliefer/add-version-to-mesos-config.
Authored-by: beliefer <be...@163.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
docs/running-on-mesos.md | 47 ++++++++++++++++---
.../org/apache/spark/deploy/mesos/config.scala | 52 +++++++++++++++++++---
2 files changed, 87 insertions(+), 12 deletions(-)
diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md
index cf51620a..6f6ae1c 100644
--- a/docs/running-on-mesos.md
+++ b/docs/running-on-mesos.md
@@ -371,7 +371,7 @@ See the [configuration page](configuration.html) for information on Spark config
#### Spark Properties
<table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
<tr>
<td><code>spark.mesos.coarse</code></td>
<td>true</td>
@@ -380,6 +380,7 @@ See the [configuration page](configuration.html) for information on Spark config
If set to <code>false</code>, runs over Mesos cluster in "fine-grained" sharing mode, where one Mesos task is created per Spark task.
Detailed information in <a href="running-on-mesos.html#mesos-run-modes">'Mesos Run Modes'</a>.
</td>
+ <td>0.6.0</td>
</tr>
<tr>
<td><code>spark.mesos.extra.cores</code></td>
@@ -391,6 +392,7 @@ See the [configuration page](configuration.html) for information on Spark config
send it more tasks. Use this to increase parallelism. This
setting is only used for Mesos coarse-grained mode.
</td>
+ <td>0.6.0</td>
</tr>
<tr>
<td><code>spark.mesos.mesosExecutor.cores</code></td>
@@ -401,6 +403,7 @@ See the [configuration page](configuration.html) for information on Spark config
is being run, each Mesos executor will occupy the number of cores configured here.
The value can be a floating point number.
</td>
+ <td>1.4.0</td>
</tr>
<tr>
<td><code>spark.mesos.executor.docker.image</code></td>
@@ -411,6 +414,7 @@ See the [configuration page](configuration.html) for information on Spark config
The installed path of Spark in the image can be specified with <code>spark.mesos.executor.home</code>;
the installed path of the Mesos library can be specified with <code>spark.executorEnv.MESOS_NATIVE_JAVA_LIBRARY</code>.
</td>
+ <td>1.4.0</td>
</tr>
<tr>
<td><code>spark.mesos.executor.docker.forcePullImage</code></td>
@@ -419,6 +423,7 @@ See the [configuration page](configuration.html) for information on Spark config
Force Mesos agents to pull the image specified in <code>spark.mesos.executor.docker.image</code>.
By default Mesos agents will not pull images they already have cached.
</td>
+ <td>2.1.0</td>
</tr>
<tr>
<td><code>spark.mesos.executor.docker.parameters</code></td>
@@ -429,6 +434,7 @@ See the [configuration page](configuration.html) for information on Spark config
<pre>key1=val1,key2=val2,key3=val3</pre>
</td>
+ <td>2.2.0</td>
</tr>
<tr>
<td><code>spark.mesos.executor.docker.volumes</code></td>
@@ -440,6 +446,7 @@ See the [configuration page](configuration.html) for information on Spark config
<pre>[host_path:]container_path[:ro|:rw]</pre>
</td>
+ <td>1.4.0</td>
</tr>
<tr>
<td><code>spark.mesos.task.labels</code></td>
@@ -450,6 +457,7 @@ See the [configuration page](configuration.html) for information on Spark config
list more than one. If your label includes a colon or comma, you
can escape it with a backslash. Ex. key:value,key2:a\:b.
</td>
+ <td>2.2.0</td>
</tr>
<tr>
<td><code>spark.mesos.executor.home</code></td>
@@ -460,6 +468,7 @@ See the [configuration page](configuration.html) for information on Spark config
them. Note that this is only relevant if a Spark binary package is not specified through
<code>spark.executor.uri</code>.
</td>
+ <td>1.1.1</td>
</tr>
<tr>
<td><code>spark.mesos.executor.memoryOverhead</code></td>
@@ -469,6 +478,7 @@ See the [configuration page](configuration.html) for information on Spark config
the overhead will be larger of either 384 or 10% of <code>spark.executor.memory</code>. If set,
the final overhead will be this value.
</td>
+ <td>1.1.1</td>
</tr>
<tr>
<td><code>spark.mesos.uris</code></td>
@@ -478,6 +488,7 @@ See the [configuration page](configuration.html) for information on Spark config
when driver or executor is launched by Mesos. This applies to
both coarse-grained and fine-grained mode.
</td>
+ <td>1.5.0</td>
</tr>
<tr>
<td><code>spark.mesos.principal</code></td>
@@ -485,6 +496,7 @@ See the [configuration page](configuration.html) for information on Spark config
<td>
Set the principal with which Spark framework will use to authenticate with Mesos. You can also specify this via the environment variable `SPARK_MESOS_PRINCIPAL`.
</td>
+ <td>1.5.0</td>
</tr>
<tr>
<td><code>spark.mesos.principal.file</code></td>
@@ -492,6 +504,7 @@ See the [configuration page](configuration.html) for information on Spark config
<td>
Set the file containing the principal with which Spark framework will use to authenticate with Mesos. Allows specifying the principal indirectly in more security conscious deployments. The file must be readable by the user launching the job and be UTF-8 encoded plaintext. You can also specify this via the environment variable `SPARK_MESOS_PRINCIPAL_FILE`.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.mesos.secret</code></td>
@@ -500,6 +513,7 @@ See the [configuration page](configuration.html) for information on Spark config
Set the secret with which Spark framework will use to authenticate with Mesos. Used, for example, when
authenticating with the registry. You can also specify this via the environment variable `SPARK_MESOS_SECRET`.
</td>
+ <td>1.5.0</td>
</tr>
<tr>
<td><code>spark.mesos.secret.file</code></td>
@@ -508,6 +522,7 @@ See the [configuration page](configuration.html) for information on Spark config
Set the file containing the secret with which Spark framework will use to authenticate with Mesos. Used, for example, when
authenticating with the registry. Allows for specifying the secret indirectly in more security conscious deployments. The file must be readable by the user launching the job and be UTF-8 encoded plaintext. You can also specify this via the environment variable `SPARK_MESOS_SECRET_FILE`.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.mesos.role</code></td>
@@ -516,6 +531,7 @@ See the [configuration page](configuration.html) for information on Spark config
Set the role of this Spark framework for Mesos. Roles are used in Mesos for reservations
and resource weight sharing.
</td>
+ <td>1.5.0</td>
</tr>
<tr>
<td><code>spark.mesos.constraints</code></td>
@@ -532,6 +548,7 @@ See the [configuration page](configuration.html) for information on Spark config
<li>In case there is no value present as a part of the constraint any offer with the corresponding attribute will be accepted (without value check).</li>
</ul>
</td>
+ <td>1.5.0</td>
</tr>
<tr>
<td><code>spark.mesos.driver.constraints</code></td>
@@ -540,6 +557,7 @@ See the [configuration page](configuration.html) for information on Spark config
Same as <code>spark.mesos.constraints</code> except applied to drivers when launched through the dispatcher. By default,
all offers with sufficient resources will be accepted.
</td>
+ <td>2.2.1</td>
</tr>
<tr>
<td><code>spark.mesos.containerizer</code></td>
@@ -550,6 +568,7 @@ See the [configuration page](configuration.html) for information on Spark config
containerizers for docker: the "docker" containerizer, and the preferred
"mesos" containerizer. Read more here: http://mesos.apache.org/documentation/latest/container-image/
</td>
+ <td>2.1.0</td>
</tr>
<tr>
<td><code>spark.mesos.driver.webui.url</code></td>
@@ -558,6 +577,7 @@ See the [configuration page](configuration.html) for information on Spark config
Set the Spark Mesos driver webui_url for interacting with the framework.
If unset it will point to Spark's internal web UI.
</td>
+ <td>2.0.0</td>
</tr>
<tr>
<td><code>spark.mesos.driver.labels</code></td>
@@ -566,8 +586,8 @@ See the [configuration page](configuration.html) for information on Spark config
Mesos labels to add to the driver. See <code>spark.mesos.task.labels</code>
for formatting information.
</td>
+ <td>2.3.0</td>
</tr>
-
<tr>
<td>
<code>spark.mesos.driver.secret.values</code>,
@@ -616,8 +636,8 @@ See the [configuration page](configuration.html) for information on Spark config
<pre>spark.mesos.driver.secret.names=password1,password2</pre>
</p>
</td>
+ <td>2.3.0</td>
</tr>
-
<tr>
<td>
<code>spark.mesos.driver.secret.envkeys</code>,
@@ -670,8 +690,8 @@ See the [configuration page](configuration.html) for information on Spark config
<pre>spark.mesos.driver.secret.filenames=pwdfile1,pwdfile2</pre>
</p>
</td>
+ <td>2.3.0</td>
</tr>
-
<tr>
<td><code>spark.mesos.driverEnv.[EnvironmentVariableName]</code></td>
<td><code>(none)</code></td>
@@ -681,6 +701,7 @@ See the [configuration page](configuration.html) for information on Spark config
driver process. The user can specify multiple of these to set
multiple environment variables.
</td>
+ <td>2.1.0</td>
</tr>
<tr>
<td><code>spark.mesos.dispatcher.webui.url</code></td>
@@ -689,7 +710,8 @@ See the [configuration page](configuration.html) for information on Spark config
Set the Spark Mesos dispatcher webui_url for interacting with the framework.
If unset it will point to Spark's internal web UI.
</td>
- </tr>
+ <td>2.0.0</td>
+</tr>
<tr>
<td><code>spark.mesos.dispatcher.driverDefault.[PropertyName]</code></td>
<td><code>(none)</code></td>
@@ -699,7 +721,8 @@ See the [configuration page](configuration.html) for information on Spark config
spark.mesos.dispatcher.driverProperty.spark.executor.memory=32g
results in the executors for all drivers submitted in cluster mode
to run in 32g containers.
-</td>
+ </td>
+ <td>2.1.0</td>
</tr>
<tr>
<td><code>spark.mesos.dispatcher.historyServer.url</code></td>
@@ -709,6 +732,7 @@ See the [configuration page](configuration.html) for information on Spark config
server</a>. The dispatcher will then link each driver to its entry
in the history server.
</td>
+ <td>2.1.0</td>
</tr>
<tr>
<td><code>spark.mesos.gpus.max</code></td>
@@ -717,7 +741,8 @@ See the [configuration page](configuration.html) for information on Spark config
Set the maximum number GPU resources to acquire for this job. Note that executors will still launch when no GPU resources are found
since this configuration is just an upper limit and not a guaranteed amount.
</td>
- </tr>
+ <td>2.1.0</td>
+</tr>
<tr>
<td><code>spark.mesos.network.name</code></td>
<td><code>(none)</code></td>
@@ -728,6 +753,7 @@ See the [configuration page](configuration.html) for information on Spark config
<a href="http://mesos.apache.org/documentation/latest/cni/">the Mesos CNI docs</a>
for more details.
</td>
+ <td>2.1.0</td>
</tr>
<tr>
<td><code>spark.mesos.network.labels</code></td>
@@ -742,6 +768,7 @@ See the [configuration page](configuration.html) for information on Spark config
<a href="http://mesos.apache.org/documentation/latest/cni/#mesos-meta-data-to-cni-plugins">the Mesos CNI docs</a>
for more details.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.mesos.fetcherCache.enable</code></td>
@@ -752,6 +779,7 @@ See the [configuration page](configuration.html) for information on Spark config
href="http://mesos.apache.org/documentation/latest/fetcher/">Mesos
Fetcher Cache</a>
</td>
+ <td>2.1.0</td>
</tr>
<tr>
<td><code>spark.mesos.driver.failoverTimeout</code></td>
@@ -763,6 +791,7 @@ See the [configuration page](configuration.html) for information on Spark config
executors. The default value is zero, meaning no timeout: if the
driver disconnects, the master immediately tears down the framework.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.mesos.rejectOfferDuration</code></td>
@@ -772,6 +801,7 @@ See the [configuration page](configuration.html) for information on Spark config
`spark.mesos.rejectOfferDurationForUnmetConstraints`,
`spark.mesos.rejectOfferDurationForReachedMaxCores`
</td>
+ <td>2.2.0</td>
</tr>
<tr>
<td><code>spark.mesos.rejectOfferDurationForUnmetConstraints</code></td>
@@ -779,6 +809,7 @@ See the [configuration page](configuration.html) for information on Spark config
<td>
Time to consider unused resources refused with unmet constraints
</td>
+ <td>1.6.0</td>
</tr>
<tr>
<td><code>spark.mesos.rejectOfferDurationForReachedMaxCores</code></td>
@@ -787,6 +818,7 @@ See the [configuration page](configuration.html) for information on Spark config
Time to consider unused resources refused when maximum number of cores
<code>spark.cores.max</code> is reached
</td>
+ <td>2.0.0</td>
</tr>
<tr>
<td><code>spark.mesos.appJar.local.resolution.mode</code></td>
@@ -799,6 +831,7 @@ See the [configuration page](configuration.html) for information on Spark config
If the value is `container` then spark submit in the container will use the jar in the container's path:
`/path/to/jar`.
</td>
+ <td>2.4.0</td>
</tr>
</table>
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
index 79a1137..e1c0d18 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
@@ -28,6 +28,7 @@ package object config {
ConfigBuilder(s"spark.mesos.$taskType.secret.names")
.doc("A comma-separated list of secret reference names. Consult the Mesos Secret " +
"protobuf for more information.")
+ .version("2.3.0")
.stringConf
.toSequence
.createOptional
@@ -35,6 +36,7 @@ package object config {
private[spark] val SECRET_VALUES =
ConfigBuilder(s"spark.mesos.$taskType.secret.values")
.doc("A comma-separated list of secret values.")
+ .version("2.3.0")
.stringConf
.toSequence
.createOptional
@@ -43,6 +45,7 @@ package object config {
ConfigBuilder(s"spark.mesos.$taskType.secret.envkeys")
.doc("A comma-separated list of the environment variables to contain the secrets." +
"The environment variable will be set on the driver.")
+ .version("2.3.0")
.stringConf
.toSequence
.createOptional
@@ -51,6 +54,7 @@ package object config {
ConfigBuilder(s"spark.mesos.$taskType.secret.filenames")
.doc("A comma-separated list of file paths secret will be written to. Consult the Mesos " +
"Secret protobuf for more information.")
+ .version("2.3.0")
.stringConf
.toSequence
.createOptional
@@ -59,6 +63,7 @@ package object config {
private[spark] val CREDENTIAL_PRINCIPAL =
ConfigBuilder("spark.mesos.principal")
.doc("Name of the Kerberos principal to authenticate Spark to Mesos.")
+ .version("1.5.0")
.stringConf
.createOptional
@@ -66,18 +71,21 @@ package object config {
ConfigBuilder("spark.mesos.principal.file")
.doc("The path of file which contains the name of the Kerberos principal " +
"to authenticate Spark to Mesos.")
+ .version("2.4.0")
.stringConf
.createOptional
private[spark] val CREDENTIAL_SECRET =
ConfigBuilder("spark.mesos.secret")
.doc("The secret value to authenticate Spark to Mesos.")
+ .version("1.5.0")
.stringConf
.createOptional
private[spark] val CREDENTIAL_SECRET_FILE =
ConfigBuilder("spark.mesos.secret.file")
.doc("The path of file which contains the secret value to authenticate Spark to Mesos.")
+ .version("2.4.0")
.stringConf
.createOptional
@@ -85,6 +93,7 @@ package object config {
private[spark] val SHUFFLE_CLEANER_INTERVAL_S =
ConfigBuilder("spark.shuffle.cleaner.interval")
+ .version("2.0.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("30s")
@@ -92,6 +101,7 @@ package object config {
ConfigBuilder("spark.mesos.dispatcher.webui.url")
.doc("Set the Spark Mesos dispatcher webui_url for interacting with the " +
"framework. If unset it will point to Spark's internal web UI.")
+ .version("2.0.0")
.stringConf
.createOptional
@@ -99,6 +109,7 @@ package object config {
ConfigBuilder("spark.mesos.dispatcher.historyServer.url")
.doc("Set the URL of the history server. The dispatcher will then " +
"link each driver to its entry in the history server.")
+ .version("2.1.0")
.stringConf
.createOptional
@@ -107,6 +118,7 @@ package object config {
.doc("Mesos labels to add to the driver. Labels are free-form key-value pairs. Key-value " +
"pairs should be separated by a colon, and commas used to list more than one." +
"Ex. key:value,key2:value2")
+ .version("2.3.0")
.stringConf
.createOptional
@@ -114,6 +126,7 @@ package object config {
ConfigBuilder("spark.mesos.driver.webui.url")
.doc("Set the Spark Mesos driver webui_url for interacting with the framework. " +
"If unset it will point to Spark's internal web UI.")
+ .version("2.0.0")
.stringConf
.createOptional
@@ -125,6 +138,7 @@ package object config {
ConfigBuilder("spark.mesos.driver.failoverTimeout")
.doc("Amount of time in seconds that the master will wait to hear from the driver, " +
"during a temporary disconnection, before tearing down all the executors.")
+ .version("2.3.0")
.doubleConf
.createWithDefault(0.0)
@@ -132,6 +146,7 @@ package object config {
ConfigBuilder("spark.mesos.network.name")
.doc("Attach containers to the given named network. If this job is launched " +
"in cluster mode, also launch the driver in the given named network.")
+ .version("2.1.0")
.stringConf
.createOptional
@@ -140,6 +155,7 @@ package object config {
.doc("Network labels to pass to CNI plugins. This is a comma-separated list " +
"of key-value pairs, where each key-value pair has the format key:value. " +
"Example: key1:val1,key2:val2")
+ .version("2.3.0")
.stringConf
.createOptional
@@ -147,19 +163,21 @@ package object config {
ConfigBuilder("spark.mesos.driver.constraints")
.doc("Attribute based constraints on mesos resource offers. Applied by the dispatcher " +
"when launching drivers. Default is to accept all offers with sufficient resources.")
+ .version("2.2.1")
.stringConf
.createWithDefault("")
private[spark] val DRIVER_FRAMEWORK_ID =
ConfigBuilder("spark.mesos.driver.frameworkId")
+ .version("2.1.0")
.stringConf
.createOptional
private[spark] val EXECUTOR_URI =
- ConfigBuilder("spark.executor.uri").stringConf.createOptional
+ ConfigBuilder("spark.executor.uri").version("0.8.0").stringConf.createOptional
private[spark] val PROXY_BASE_URL =
- ConfigBuilder("spark.mesos.proxy.baseURL").stringConf.createOptional
+ ConfigBuilder("spark.mesos.proxy.baseURL").version("2.3.0").stringConf.createOptional
private[spark] val COARSE_MODE =
ConfigBuilder("spark.mesos.coarse")
@@ -167,22 +185,26 @@ package object config {
"Spark acquires one long-lived Mesos task on each machine. If set to false, runs over " +
"Mesos cluster in \"fine-grained\" sharing mode, where one Mesos task is created per " +
"Spark task.")
- .booleanConf.createWithDefault(true)
+ .version("0.6.0")
+ .booleanConf
+ .createWithDefault(true)
private[spark] val COARSE_SHUTDOWN_TIMEOUT =
ConfigBuilder("spark.mesos.coarse.shutdownTimeout")
+ .version("2.0.0")
.timeConf(TimeUnit.MILLISECONDS)
.checkValue(_ >= 0, s"spark.mesos.coarse.shutdownTimeout must be >= 0")
.createWithDefaultString("10s")
private[spark] val MAX_DRIVERS =
- ConfigBuilder("spark.mesos.maxDrivers").intConf.createWithDefault(200)
+ ConfigBuilder("spark.mesos.maxDrivers").version("1.4.0").intConf.createWithDefault(200)
private[spark] val RETAINED_DRIVERS =
- ConfigBuilder("spark.mesos.retainedDrivers").intConf.createWithDefault(200)
+ ConfigBuilder("spark.mesos.retainedDrivers").version("1.4.0").intConf.createWithDefault(200)
private[spark] val CLUSTER_RETRY_WAIT_MAX_SECONDS =
ConfigBuilder("spark.mesos.cluster.retry.wait.max")
+ .version("1.4.0")
.intConf
.createWithDefault(60) // 1 minute
@@ -190,6 +212,7 @@ package object config {
ConfigBuilder("spark.mesos.fetcherCache.enable")
.doc("If set to true, all URIs (example: `spark.executor.uri`, `spark.mesos.uris`) will be " +
"cached by the Mesos Fetcher Cache.")
+ .version("2.1.0")
.booleanConf
.createWithDefault(false)
@@ -201,6 +224,7 @@ package object config {
"resource from the host's file system. If the value is unknown it prints a warning msg " +
"in the dispatcher logs and defaults to `host`. If the value is `container` then spark " +
"submit in the container will use the jar in the container's path: `/path/to/jar`.")
+ .version("2.4.0")
.stringConf
.checkValues(Set("host", "container"))
.createWithDefault("host")
@@ -210,12 +234,14 @@ package object config {
.doc("Time to consider unused resources refused, serves as a fallback of " +
"`spark.mesos.rejectOfferDurationForUnmetConstraints`, " +
"`spark.mesos.rejectOfferDurationForReachedMaxCores`.")
+ .version("2.2.0")
.timeConf(TimeUnit.SECONDS)
.createWithDefaultString("120s")
private[spark] val REJECT_OFFER_DURATION_FOR_UNMET_CONSTRAINTS =
ConfigBuilder("spark.mesos.rejectOfferDurationForUnmetConstraints")
.doc("Time to consider unused resources refused with unmet constraints.")
+ .version("1.6.0")
.timeConf(TimeUnit.SECONDS)
.createOptional
@@ -223,6 +249,7 @@ package object config {
ConfigBuilder("spark.mesos.rejectOfferDurationForReachedMaxCores")
.doc("Time to consider unused resources refused when maximum number of cores " +
"`spark.cores.max` is reached.")
+ .version("2.0.0")
.timeConf(TimeUnit.SECONDS)
.createOptional
@@ -231,6 +258,7 @@ package object config {
.doc("A comma-separated list of URIs to be downloaded to the sandbox when driver or " +
"executor is launched by Mesos. This applies to both coarse-grained and fine-grained " +
"mode.")
+ .version("1.5.0")
.stringConf
.toSequence
.createWithDefault(Nil)
@@ -241,6 +269,7 @@ package object config {
"By default, the executors will simply use the driver's Spark home directory, which may " +
"not be visible to them. Note that this is only relevant if a Spark binary package is " +
"not specified through `spark.executor.uri`.")
+ .version("1.1.1")
.stringConf
.createOptional
@@ -250,6 +279,7 @@ package object config {
"include the cores used to run the Spark tasks. In other words, even if no Spark task " +
"is being run, each Mesos executor will occupy the number of cores configured here. " +
"The value can be a floating point number.")
+ .version("1.4.0")
.doubleConf
.createWithDefault(1.0)
@@ -259,6 +289,7 @@ package object config {
"more cores allocated. It instead means that an executor will \"pretend\" it has more " +
"cores, so that the driver will send it more tasks. Use this to increase parallelism. " +
"This setting is only used for Mesos coarse-grained mode.")
+ .version("0.6.0")
.intConf
.createWithDefault(0)
@@ -267,6 +298,7 @@ package object config {
.doc("The amount of additional memory, specified in MiB, to be allocated per executor. " +
"By default, the overhead will be larger of either 384 or 10% of " +
"`spark.executor.memory`. If set, the final overhead will be this value.")
+ .version("1.1.1")
.intConf
.createOptional
@@ -277,6 +309,7 @@ package object config {
"The installed path of Spark in the image can be specified with " +
"`spark.mesos.executor.home`; the installed path of the Mesos library can be specified " +
"with `spark.executorEnv.MESOS_NATIVE_JAVA_LIBRARY`.")
+ .version("1.4.0")
.stringConf
.createOptional
@@ -285,11 +318,13 @@ package object config {
.doc("Force Mesos agents to pull the image specified in " +
"`spark.mesos.executor.docker.image`. By default Mesos agents will not pull images they " +
"already have cached.")
+ .version("2.1.0")
.booleanConf
.createOptional
private[spark] val EXECUTOR_DOCKER_PORT_MAPS =
ConfigBuilder("spark.mesos.executor.docker.portmaps")
+ .version("1.4.0")
.stringConf
.toSequence
.createOptional
@@ -299,6 +334,7 @@ package object config {
.doc("Set the list of custom parameters which will be passed into the `docker run` " +
"command when launching the Spark executor on Mesos using the docker containerizer. " +
"The format of this property is a list of key/value pairs which pair looks key1=value1.")
+ .version("2.2.0")
.stringConf
.toSequence
.createOptional
@@ -309,6 +345,7 @@ package object config {
"using `spark.mesos.executor.docker.image`. The format of this property is a list of " +
"mappings following the form passed to `docker run -v`. That is they take the form: " +
"`[host_path:]container_path[:ro|:rw]`")
+ .version("1.4.0")
.stringConf
.toSequence
.createOptional
@@ -318,6 +355,7 @@ package object config {
.doc("Set the maximum number GPU resources to acquire for this job. Note that executors " +
"will still launch when no GPU resources are found since this configuration is just an " +
"upper limit and not a guaranteed amount.")
+ .version("2.1.0")
.intConf
.createWithDefault(0)
@@ -327,6 +365,7 @@ package object config {
"Key-value pairs should be separated by a colon, and commas used to list more than one. " +
"If your label includes a colon or comma, you can escape it with a backslash. " +
"Ex. key:value,key2:a\\:b.")
+ .version("2.2.0")
.stringConf
.createWithDefault("")
@@ -335,6 +374,7 @@ package object config {
.doc("Attribute-based constraints on mesos resource offers. By default, all resource " +
"offers will be accepted. This setting applies only to executors. Refer to Mesos " +
"Attributes & Resources doc for more information on attributes.")
+ .version("1.5.0")
.stringConf
.createWithDefault("")
@@ -344,6 +384,7 @@ package object config {
"Mesos supports two types of containerizers for docker: the \"docker\" containerizer, " +
"and the preferred \"mesos\" containerizer. " +
"Read more here: http://mesos.apache.org/documentation/latest/container-image/")
+ .version("2.1.0")
.stringConf
.checkValues(Set("docker", "mesos"))
.createWithDefault("docker")
@@ -352,6 +393,7 @@ package object config {
ConfigBuilder("spark.mesos.role")
.doc("Set the role of this Spark framework for Mesos. Roles are used in Mesos for " +
"reservations and resource weight sharing.")
+ .version("1.5.0")
.stringConf
.createOptional
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org