You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2015/04/02 21:18:43 UTC
spark git commit: [SPARK-6667] [PySpark] remove setReuseAddress
Repository: spark
Updated Branches:
refs/heads/master 424e987df -> 0cce5451a
[SPARK-6667] [PySpark] remove setReuseAddress
The reused address on server side had caused the server can not acknowledge the connected connections, remove it.
This PR will retry once after timeout, it also add a timeout at client side.
Author: Davies Liu <da...@databricks.com>
Closes #5324 from davies/collect_hang and squashes the following commits:
e5a51a2 [Davies Liu] remove setReuseAddress
7977c2f [Davies Liu] do retry on client side
b838f35 [Davies Liu] retry after timeout
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0cce5451
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0cce5451
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0cce5451
Branch: refs/heads/master
Commit: 0cce5451adfc6bf4661bcf67aca3db26376455fe
Parents: 424e987
Author: Davies Liu <da...@databricks.com>
Authored: Thu Apr 2 12:18:33 2015 -0700
Committer: Josh Rosen <jo...@databricks.com>
Committed: Thu Apr 2 12:18:33 2015 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala | 1 -
python/pyspark/rdd.py | 1 +
2 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0cce5451/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
index 19f4c95..36cf2af 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
@@ -605,7 +605,6 @@ private[spark] object PythonRDD extends Logging {
*/
private def serveIterator[T](items: Iterator[T], threadName: String): Int = {
val serverSocket = new ServerSocket(0, 1)
- serverSocket.setReuseAddress(true)
// Close the socket if no connection in 3 seconds
serverSocket.setSoTimeout(3000)
http://git-wip-us.apache.org/repos/asf/spark/blob/0cce5451/python/pyspark/rdd.py
----------------------------------------------------------------------
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
index c337a43..2d05611 100644
--- a/python/pyspark/rdd.py
+++ b/python/pyspark/rdd.py
@@ -113,6 +113,7 @@ def _parse_memory(s):
def _load_from_socket(port, serializer):
sock = socket.socket()
+ sock.settimeout(3)
try:
sock.connect(("localhost", port))
rf = sock.makefile("rb", 65536)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org