You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/06/30 03:02:27 UTC
spark git commit: [SPARK-21253][CORE] Disable
spark.reducer.maxReqSizeShuffleToMem
Repository: spark
Updated Branches:
refs/heads/master 4996c5394 -> 80f7ac3a6
[SPARK-21253][CORE] Disable spark.reducer.maxReqSizeShuffleToMem
## What changes were proposed in this pull request?
Disable spark.reducer.maxReqSizeShuffleToMem because it breaks the old shuffle service.
Credits to wangyum
Closes #18466
## How was this patch tested?
Jenkins
Author: Shixiong Zhu <sh...@databricks.com>
Author: Yuming Wang <wg...@gmail.com>
Closes #18467 from zsxwing/SPARK-21253.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/80f7ac3a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/80f7ac3a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/80f7ac3a
Branch: refs/heads/master
Commit: 80f7ac3a601709dd9471092244612023363f54cd
Parents: 4996c53
Author: Shixiong Zhu <sh...@databricks.com>
Authored: Fri Jun 30 11:02:22 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Fri Jun 30 11:02:22 2017 +0800
----------------------------------------------------------------------
.../scala/org/apache/spark/internal/config/package.scala | 3 ++-
docs/configuration.md | 8 --------
2 files changed, 2 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/80f7ac3a/core/src/main/scala/org/apache/spark/internal/config/package.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index be63c63..8dee0d9 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -323,10 +323,11 @@ package object config {
private[spark] val REDUCER_MAX_REQ_SIZE_SHUFFLE_TO_MEM =
ConfigBuilder("spark.reducer.maxReqSizeShuffleToMem")
+ .internal()
.doc("The blocks of a shuffle request will be fetched to disk when size of the request is " +
"above this threshold. This is to avoid a giant request takes too much memory.")
.bytesConf(ByteUnit.BYTE)
- .createWithDefaultString("200m")
+ .createWithDefault(Long.MaxValue)
private[spark] val TASK_METRICS_TRACK_UPDATED_BLOCK_STATUSES =
ConfigBuilder("spark.taskMetrics.trackUpdatedBlockStatuses")
http://git-wip-us.apache.org/repos/asf/spark/blob/80f7ac3a/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index c8e6153..bd6a1f9 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -529,14 +529,6 @@ Apart from these, the following properties are also available, and may be useful
</td>
</tr>
<tr>
- <td><code>spark.reducer.maxReqSizeShuffleToMem</code></td>
- <td>200m</td>
- <td>
- The blocks of a shuffle request will be fetched to disk when size of the request is above
- this threshold. This is to avoid a giant request takes too much memory.
- </td>
-</tr>
-<tr>
<td><code>spark.shuffle.compress</code></td>
<td>true</td>
<td>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org