You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/06/01 00:10:43 UTC
git commit: SPARK-1917: fix PySpark import of scipy.special functions
Repository: spark
Updated Branches:
refs/heads/branch-0.9 563bfe1dd -> e03af416c
SPARK-1917: fix PySpark import of scipy.special functions
https://issues.apache.org/jira/browse/SPARK-1917
Author: Uri Laserson <la...@cloudera.com>
Closes #866 from laserson/SPARK-1917 and squashes the following commits:
d947e8c [Uri Laserson] Added test for scipy.special importing
1798bbd [Uri Laserson] SPARK-1917: fix PySpark import of scipy.special
Conflicts:
python/pyspark/tests.py
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e03af416
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e03af416
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e03af416
Branch: refs/heads/branch-0.9
Commit: e03af416cf6947359135299e5550108ecb8001b2
Parents: 563bfe1
Author: Uri Laserson <la...@cloudera.com>
Authored: Sat May 31 14:59:09 2014 -0700
Committer: Matei Zaharia <ma...@databricks.com>
Committed: Sat May 31 15:08:00 2014 -0700
----------------------------------------------------------------------
python/pyspark/cloudpickle.py | 2 +-
python/pyspark/tests.py | 25 +++++++++++++++++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e03af416/python/pyspark/cloudpickle.py
----------------------------------------------------------------------
diff --git a/python/pyspark/cloudpickle.py b/python/pyspark/cloudpickle.py
index 6a7c23a..eb5dbb8 100644
--- a/python/pyspark/cloudpickle.py
+++ b/python/pyspark/cloudpickle.py
@@ -933,7 +933,7 @@ def _change_cell_value(cell, newval):
Note: These can never be renamed due to client compatibility issues"""
def _getobject(modname, attribute):
- mod = __import__(modname)
+ mod = __import__(modname, fromlist=[attribute])
return mod.__dict__[attribute]
def _generateImage(size, mode, str_rep):
http://git-wip-us.apache.org/repos/asf/spark/blob/e03af416/python/pyspark/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py
index 5271045..c741bfb 100644
--- a/python/pyspark/tests.py
+++ b/python/pyspark/tests.py
@@ -33,6 +33,14 @@ from pyspark.files import SparkFiles
from pyspark.java_gateway import SPARK_HOME
from pyspark.serializers import read_int
+_have_scipy = False
+try:
+ import scipy.sparse
+ _have_scipy = True
+except:
+ # No SciPy, but that's okay, we'll skip those tests
+ pass
+
class PySparkTestCase(unittest.TestCase):
@@ -234,5 +242,22 @@ class TestDaemon(unittest.TestCase):
from signal import SIGTERM
self.do_termination_test(lambda daemon: os.kill(daemon.pid, SIGTERM))
+
+@unittest.skipIf(not _have_scipy, "SciPy not installed")
+class SciPyTests(PySparkTestCase):
+ """General PySpark tests that depend on scipy """
+
+ def test_serialize(self):
+ from scipy.special import gammaln
+ x = range(1, 5)
+ expected = map(gammaln, x)
+ observed = self.sc.parallelize(x).map(gammaln).collect()
+ self.assertEqual(expected, observed)
+
+
if __name__ == "__main__":
+ if not _have_scipy:
+ print "NOTE: Skipping SciPy tests as it does not seem to be installed"
unittest.main()
+ if not _have_scipy:
+ print "NOTE: SciPy tests were skipped as it does not seem to be installed"