You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2014/01/26 07:41:36 UTC
[1/2] git commit: Fix for SPARK-1025: PySpark hang on missing files.
Updated Branches:
refs/heads/master c66a2ef1c -> c40619d48
Fix for SPARK-1025: PySpark hang on missing files.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/f8306849
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/f8306849
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/f8306849
Branch: refs/heads/master
Commit: f83068497ba42c5ea5c636efebca81f684e96177
Parents: 6156990
Author: Josh Rosen <jo...@apache.org>
Authored: Thu Jan 23 18:10:16 2014 -0800
Committer: Josh Rosen <jo...@apache.org>
Committed: Thu Jan 23 18:24:51 2014 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/api/python/PythonRDD.scala | 9 +++++++++
python/pyspark/tests.py | 11 +++++++++++
2 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/f8306849/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
index 57bde8d..70516bd 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
@@ -52,6 +52,8 @@ private[spark] class PythonRDD[T: ClassTag](
val env = SparkEnv.get
val worker = env.createPythonWorker(pythonExec, envVars.toMap)
+ @volatile var readerException: Exception = null
+
// Start a thread to feed the process input from our parent's iterator
new Thread("stdin writer for " + pythonExec) {
override def run() {
@@ -82,6 +84,10 @@ private[spark] class PythonRDD[T: ClassTag](
dataOut.flush()
worker.shutdownOutput()
} catch {
+ case e: java.io.FileNotFoundException =>
+ readerException = e
+ // Kill the Python worker process:
+ worker.shutdownOutput()
case e: IOException =>
// This can happen for legitimate reasons if the Python code stops returning data before we are done
// passing elements through, e.g., for take(). Just log a message to say it happened.
@@ -106,6 +112,9 @@ private[spark] class PythonRDD[T: ClassTag](
}
private def read(): Array[Byte] = {
+ if (readerException != null) {
+ throw readerException
+ }
try {
stream.readInt() match {
case length if length > 0 =>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/f8306849/python/pyspark/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py
index acd1ca5..5271045 100644
--- a/python/pyspark/tests.py
+++ b/python/pyspark/tests.py
@@ -168,6 +168,17 @@ class TestRDDFunctions(PySparkTestCase):
self.assertEqual("Hello World!", x.strip())
self.assertEqual("Hello World!", y.strip())
+ def test_deleting_input_files(self):
+ # Regression test for SPARK-1025
+ tempFile = NamedTemporaryFile(delete=False)
+ tempFile.write("Hello World!")
+ tempFile.close()
+ data = self.sc.textFile(tempFile.name)
+ filtered_data = data.filter(lambda x: True)
+ self.assertEqual(1, filtered_data.count())
+ os.unlink(tempFile.name)
+ self.assertRaises(Exception, lambda: filtered_data.count())
+
class TestIO(PySparkTestCase):
[2/2] git commit: Merge pull request #504 from JoshRosen/SPARK-1025
Posted by rx...@apache.org.
Merge pull request #504 from JoshRosen/SPARK-1025
Fix PySpark hang when input files are deleted (SPARK-1025)
This pull request addresses [SPARK-1025](https://spark-project.atlassian.net/browse/SPARK-1025), an issue where PySpark could hang if its input files were deleted.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/c40619d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/c40619d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/c40619d4
Branch: refs/heads/master
Commit: c40619d4873f36ffb96a2e6292b32d5b64eab153
Parents: c66a2ef f830684
Author: Reynold Xin <rx...@apache.org>
Authored: Sat Jan 25 22:41:30 2014 -0800
Committer: Reynold Xin <rx...@apache.org>
Committed: Sat Jan 25 22:41:30 2014 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/api/python/PythonRDD.scala | 9 +++++++++
python/pyspark/tests.py | 11 +++++++++++
2 files changed, 20 insertions(+)
----------------------------------------------------------------------