You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/01/06 00:19:57 UTC
spark git commit: [SPARK-5093] Set spark.network.timeout to 120s
consistently.
Repository: spark
Updated Branches:
refs/heads/master 6c6f32574 -> bbcba3a94
[SPARK-5093] Set spark.network.timeout to 120s consistently.
Author: Reynold Xin <rx...@databricks.com>
Closes #3903 from rxin/timeout-120 and squashes the following commits:
7c2138e [Reynold Xin] [SPARK-5093] Set spark.network.timeout to 120s consistently.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bbcba3a9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bbcba3a9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bbcba3a9
Branch: refs/heads/master
Commit: bbcba3a9430365640c0188e7ca6e0677d3227dd8
Parents: 6c6f325
Author: Reynold Xin <rx...@databricks.com>
Authored: Mon Jan 5 15:19:53 2015 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Mon Jan 5 15:19:53 2015 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/network/nio/ConnectionManager.scala | 2 +-
.../org/apache/spark/storage/BlockManagerMasterActor.scala | 6 +-----
core/src/main/scala/org/apache/spark/util/AkkaUtils.scala | 2 +-
docs/configuration.md | 6 +++---
.../main/java/org/apache/spark/network/util/TransportConf.java | 5 ++---
5 files changed, 8 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bbcba3a9/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala b/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala
index 98455c0..3340fca 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala
@@ -82,7 +82,7 @@ private[nio] class ConnectionManager(
new HashedWheelTimer(Utils.namedThreadFactory("AckTimeoutMonitor"))
private val ackTimeout =
- conf.getInt("spark.core.connection.ack.wait.timeout", conf.getInt("spark.network.timeout", 100))
+ conf.getInt("spark.core.connection.ack.wait.timeout", conf.getInt("spark.network.timeout", 120))
// Get the thread counts from the Spark Configuration.
//
http://git-wip-us.apache.org/repos/asf/spark/blob/bbcba3a9/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala
index 9d77cf2..6413346 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala
@@ -52,11 +52,7 @@ class BlockManagerMasterActor(val isLocal: Boolean, conf: SparkConf, listenerBus
private val akkaTimeout = AkkaUtils.askTimeout(conf)
- val slaveTimeout = {
- val defaultMs = math.max(conf.getInt("spark.executor.heartbeatInterval", 10000) * 3, 45000)
- val networkTimeout = conf.getInt("spark.network.timeout", defaultMs / 1000)
- conf.getLong("spark.storage.blockManagerSlaveTimeoutMs", networkTimeout * 1000)
- }
+ val slaveTimeout = conf.getLong("spark.storage.blockManagerSlaveTimeoutMs", 120 * 1000)
val checkTimeoutInterval = conf.getLong("spark.storage.blockManagerTimeoutIntervalMs", 60000)
http://git-wip-us.apache.org/repos/asf/spark/blob/bbcba3a9/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala b/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala
index 8d86fd3..db2531d 100644
--- a/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala
@@ -65,7 +65,7 @@ private[spark] object AkkaUtils extends Logging {
val akkaThreads = conf.getInt("spark.akka.threads", 4)
val akkaBatchSize = conf.getInt("spark.akka.batchSize", 15)
- val akkaTimeout = conf.getInt("spark.akka.timeout", conf.getInt("spark.network.timeout", 100))
+ val akkaTimeout = conf.getInt("spark.akka.timeout", conf.getInt("spark.network.timeout", 120))
val akkaFrameSize = maxFrameSizeBytes(conf)
val akkaLogLifecycleEvents = conf.getBoolean("spark.akka.logLifecycleEvents", false)
val lifecycleEvents = if (akkaLogLifecycleEvents) "on" else "off"
http://git-wip-us.apache.org/repos/asf/spark/blob/bbcba3a9/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index 7ada67f..2add485 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -820,12 +820,12 @@ Apart from these, the following properties are also available, and may be useful
</tr>
<tr>
<td><code>spark.network.timeout</code></td>
- <td>100</td>
+ <td>120</td>
<td>
Default timeout for all network interactions, in seconds. This config will be used in
place of <code>spark.core.connection.ack.wait.timeout</code>, <code>spark.akka.timeout</code>,
- <code>spark.storage.blockManagerSlaveTimeoutMs</code> or <code>spark.shuffle.io.connectionTimeout</code>,
- if they are not configured.
+ <code>spark.storage.blockManagerSlaveTimeoutMs</code> or
+ <code>spark.shuffle.io.connectionTimeout</code>, if they are not configured.
</td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/spark/blob/bbcba3a9/network/common/src/main/java/org/apache/spark/network/util/TransportConf.java
----------------------------------------------------------------------
diff --git a/network/common/src/main/java/org/apache/spark/network/util/TransportConf.java b/network/common/src/main/java/org/apache/spark/network/util/TransportConf.java
index e34382d..6c91786 100644
--- a/network/common/src/main/java/org/apache/spark/network/util/TransportConf.java
+++ b/network/common/src/main/java/org/apache/spark/network/util/TransportConf.java
@@ -37,9 +37,8 @@ public class TransportConf {
/** Connect timeout in milliseconds. Default 120 secs. */
public int connectionTimeoutMs() {
- int timeout =
- conf.getInt("spark.shuffle.io.connectionTimeout", conf.getInt("spark.network.timeout", 100));
- return timeout * 1000;
+ int defaultTimeout = conf.getInt("spark.network.timeout", 120);
+ return conf.getInt("spark.shuffle.io.connectionTimeout", defaultTimeout) * 1000;
}
/** Number of concurrent connections between two nodes for fetching data. */
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org