You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2015/01/13 01:16:47 UTC
spark git commit: [SPARK-4348] [SPARK-4821] Backport PySpark
random.py -> rand.py fix to branch-1.1
Repository: spark
Updated Branches:
refs/heads/branch-1.1 ee33699e9 -> c98dc0e8b
[SPARK-4348] [SPARK-4821] Backport PySpark random.py -> rand.py fix to branch-1.1
This backports #3216 and #3669 to `branch-1.1` in order to fix the PySpark unit tests.
Author: Joseph K. Bradley <jo...@databricks.com>
Author: Davies Liu <da...@databricks.com>
Closes #4011 from JoshRosen/pyspark-rand-fix-1.1-backport and squashes the following commits:
ace4cb6 [Joseph K. Bradley] [SPARK-4821] [mllib] [python] [docs] Fix for pyspark.mllib.rand doc
7ae5a1c [Davies Liu] [SPARK-4348] [PySpark] [MLlib] rename random.py to rand.py
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c98dc0e8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c98dc0e8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c98dc0e8
Branch: refs/heads/branch-1.1
Commit: c98dc0e8ba46793c0803012bcda231d5fdb79c7d
Parents: ee33699
Author: Joseph K. Bradley <jo...@databricks.com>
Authored: Mon Jan 12 16:16:36 2015 -0800
Committer: Josh Rosen <jo...@databricks.com>
Committed: Mon Jan 12 16:16:36 2015 -0800
----------------------------------------------------------------------
python/pyspark/__init__.py | 10 --
python/pyspark/mllib/__init__.py | 9 ++
python/pyspark/mllib/linalg.py | 4 -
python/pyspark/mllib/rand.py | 183 ++++++++++++++++++++++++++++++++++
python/pyspark/mllib/random.py | 183 ----------------------------------
python/run-tests | 2 +-
6 files changed, 193 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c98dc0e8/python/pyspark/__init__.py
----------------------------------------------------------------------
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index c58555f..312c75d 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -49,16 +49,6 @@ Hive:
Main entry point for accessing data stored in Apache Hive..
"""
-# The following block allows us to import python's random instead of mllib.random for scripts in
-# mllib that depend on top level pyspark packages, which transitively depend on python's random.
-# Since Python's import logic looks for modules in the current package first, we eliminate
-# mllib.random as a candidate for C{import random} by removing the first search path, the script's
-# location, in order to force the loader to look in Python's top-level modules for C{random}.
-import sys
-s = sys.path.pop(0)
-import random
-sys.path.insert(0, s)
-
from pyspark.conf import SparkConf
from pyspark.context import SparkContext
from pyspark.sql import SQLContext
http://git-wip-us.apache.org/repos/asf/spark/blob/c98dc0e8/python/pyspark/mllib/__init__.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/__init__.py b/python/pyspark/mllib/__init__.py
index 4149f54..0c7f491 100644
--- a/python/pyspark/mllib/__init__.py
+++ b/python/pyspark/mllib/__init__.py
@@ -24,3 +24,12 @@ Python bindings for MLlib.
import numpy
if numpy.version.version < '1.4':
raise Exception("MLlib requires NumPy 1.4+")
+
+__all__ = ['classification', 'clustering', 'linalg', 'random',
+ 'recommendation', 'regression', 'stat', 'tree', 'util']
+
+import sys
+import rand as random
+random.__name__ = 'random'
+random.RandomRDDs.__module__ = __name__ + '.random'
+sys.modules[__name__ + '.random'] = random
http://git-wip-us.apache.org/repos/asf/spark/blob/c98dc0e8/python/pyspark/mllib/linalg.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/linalg.py b/python/pyspark/mllib/linalg.py
index f485a69..21d4624 100644
--- a/python/pyspark/mllib/linalg.py
+++ b/python/pyspark/mllib/linalg.py
@@ -267,8 +267,4 @@ def _test():
exit(-1)
if __name__ == "__main__":
- # remove current path from list of search paths to avoid importing mllib.random
- # for C{import random}, which is done in an external dependency of pyspark during doctests.
- import sys
- sys.path.pop(0)
_test()
http://git-wip-us.apache.org/repos/asf/spark/blob/c98dc0e8/python/pyspark/mllib/rand.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/rand.py b/python/pyspark/mllib/rand.py
new file mode 100644
index 0000000..4dc1a4a
--- /dev/null
+++ b/python/pyspark/mllib/rand.py
@@ -0,0 +1,183 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Python package for random data generation.
+"""
+
+
+from pyspark.rdd import RDD
+from pyspark.mllib._common import _deserialize_double, _deserialize_double_vector
+from pyspark.serializers import NoOpSerializer
+
+
+class RandomRDDs:
+ """
+ Generator methods for creating RDDs comprised of i.i.d samples from
+ some distribution.
+ """
+
+ @staticmethod
+ def uniformRDD(sc, size, numPartitions=None, seed=None):
+ """
+ Generates an RDD comprised of i.i.d. samples from the
+ uniform distribution U(0.0, 1.0).
+
+ To transform the distribution in the generated RDD from U(0.0, 1.0)
+ to U(a, b), use
+ C{RandomRDDs.uniformRDD(sc, n, p, seed)\
+ .map(lambda v: a + (b - a) * v)}
+
+ >>> x = RandomRDDs.uniformRDD(sc, 100).collect()
+ >>> len(x)
+ 100
+ >>> max(x) <= 1.0 and min(x) >= 0.0
+ True
+ >>> RandomRDDs.uniformRDD(sc, 100, 4).getNumPartitions()
+ 4
+ >>> parts = RandomRDDs.uniformRDD(sc, 100, seed=4).getNumPartitions()
+ >>> parts == sc.defaultParallelism
+ True
+ """
+ jrdd = sc._jvm.PythonMLLibAPI().uniformRDD(sc._jsc, size, numPartitions, seed)
+ uniform = RDD(jrdd, sc, NoOpSerializer())
+ return uniform.map(lambda bytes: _deserialize_double(bytearray(bytes)))
+
+ @staticmethod
+ def normalRDD(sc, size, numPartitions=None, seed=None):
+ """
+ Generates an RDD comprised of i.i.d. samples from the standard normal
+ distribution.
+
+ To transform the distribution in the generated RDD from standard normal
+ to some other normal N(mean, sigma^2), use
+ C{RandomRDDs.normal(sc, n, p, seed)\
+ .map(lambda v: mean + sigma * v)}
+
+ >>> x = RandomRDDs.normalRDD(sc, 1000, seed=1L)
+ >>> stats = x.stats()
+ >>> stats.count()
+ 1000L
+ >>> abs(stats.mean() - 0.0) < 0.1
+ True
+ >>> abs(stats.stdev() - 1.0) < 0.1
+ True
+ """
+ jrdd = sc._jvm.PythonMLLibAPI().normalRDD(sc._jsc, size, numPartitions, seed)
+ normal = RDD(jrdd, sc, NoOpSerializer())
+ return normal.map(lambda bytes: _deserialize_double(bytearray(bytes)))
+
+ @staticmethod
+ def poissonRDD(sc, mean, size, numPartitions=None, seed=None):
+ """
+ Generates an RDD comprised of i.i.d. samples from the Poisson
+ distribution with the input mean.
+
+ >>> mean = 100.0
+ >>> x = RandomRDDs.poissonRDD(sc, mean, 1000, seed=1L)
+ >>> stats = x.stats()
+ >>> stats.count()
+ 1000L
+ >>> abs(stats.mean() - mean) < 0.5
+ True
+ >>> from math import sqrt
+ >>> abs(stats.stdev() - sqrt(mean)) < 0.5
+ True
+ """
+ jrdd = sc._jvm.PythonMLLibAPI().poissonRDD(sc._jsc, mean, size, numPartitions, seed)
+ poisson = RDD(jrdd, sc, NoOpSerializer())
+ return poisson.map(lambda bytes: _deserialize_double(bytearray(bytes)))
+
+ @staticmethod
+ def uniformVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
+ """
+ Generates an RDD comprised of vectors containing i.i.d. samples drawn
+ from the uniform distribution U(0.0, 1.0).
+
+ >>> import numpy as np
+ >>> mat = np.matrix(RandomRDDs.uniformVectorRDD(sc, 10, 10).collect())
+ >>> mat.shape
+ (10, 10)
+ >>> mat.max() <= 1.0 and mat.min() >= 0.0
+ True
+ >>> RandomRDDs.uniformVectorRDD(sc, 10, 10, 4).getNumPartitions()
+ 4
+ """
+ jrdd = sc._jvm.PythonMLLibAPI() \
+ .uniformVectorRDD(sc._jsc, numRows, numCols, numPartitions, seed)
+ uniform = RDD(jrdd, sc, NoOpSerializer())
+ return uniform.map(lambda bytes: _deserialize_double_vector(bytearray(bytes)))
+
+ @staticmethod
+ def normalVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
+ """
+ Generates an RDD comprised of vectors containing i.i.d. samples drawn
+ from the standard normal distribution.
+
+ >>> import numpy as np
+ >>> mat = np.matrix(RandomRDDs.normalVectorRDD(sc, 100, 100, seed=1L).collect())
+ >>> mat.shape
+ (100, 100)
+ >>> abs(mat.mean() - 0.0) < 0.1
+ True
+ >>> abs(mat.std() - 1.0) < 0.1
+ True
+ """
+ jrdd = sc._jvm.PythonMLLibAPI() \
+ .normalVectorRDD(sc._jsc, numRows, numCols, numPartitions, seed)
+ normal = RDD(jrdd, sc, NoOpSerializer())
+ return normal.map(lambda bytes: _deserialize_double_vector(bytearray(bytes)))
+
+ @staticmethod
+ def poissonVectorRDD(sc, mean, numRows, numCols, numPartitions=None, seed=None):
+ """
+ Generates an RDD comprised of vectors containing i.i.d. samples drawn
+ from the Poisson distribution with the input mean.
+
+ >>> import numpy as np
+ >>> mean = 100.0
+ >>> rdd = RandomRDDs.poissonVectorRDD(sc, mean, 100, 100, seed=1L)
+ >>> mat = np.mat(rdd.collect())
+ >>> mat.shape
+ (100, 100)
+ >>> abs(mat.mean() - mean) < 0.5
+ True
+ >>> from math import sqrt
+ >>> abs(mat.std() - sqrt(mean)) < 0.5
+ True
+ """
+ jrdd = sc._jvm.PythonMLLibAPI() \
+ .poissonVectorRDD(sc._jsc, mean, numRows, numCols, numPartitions, seed)
+ poisson = RDD(jrdd, sc, NoOpSerializer())
+ return poisson.map(lambda bytes: _deserialize_double_vector(bytearray(bytes)))
+
+
+def _test():
+ import doctest
+ from pyspark.context import SparkContext
+ globs = globals().copy()
+ # The small batch size here ensures that we see multiple batches,
+ # even in these small test examples:
+ globs['sc'] = SparkContext('local[2]', 'PythonTest', batchSize=2)
+ (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ globs['sc'].stop()
+ if failure_count:
+ exit(-1)
+
+
+if __name__ == "__main__":
+ _test()
http://git-wip-us.apache.org/repos/asf/spark/blob/c98dc0e8/python/pyspark/mllib/random.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/random.py b/python/pyspark/mllib/random.py
deleted file mode 100644
index 4dc1a4a..0000000
--- a/python/pyspark/mllib/random.py
+++ /dev/null
@@ -1,183 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-Python package for random data generation.
-"""
-
-
-from pyspark.rdd import RDD
-from pyspark.mllib._common import _deserialize_double, _deserialize_double_vector
-from pyspark.serializers import NoOpSerializer
-
-
-class RandomRDDs:
- """
- Generator methods for creating RDDs comprised of i.i.d samples from
- some distribution.
- """
-
- @staticmethod
- def uniformRDD(sc, size, numPartitions=None, seed=None):
- """
- Generates an RDD comprised of i.i.d. samples from the
- uniform distribution U(0.0, 1.0).
-
- To transform the distribution in the generated RDD from U(0.0, 1.0)
- to U(a, b), use
- C{RandomRDDs.uniformRDD(sc, n, p, seed)\
- .map(lambda v: a + (b - a) * v)}
-
- >>> x = RandomRDDs.uniformRDD(sc, 100).collect()
- >>> len(x)
- 100
- >>> max(x) <= 1.0 and min(x) >= 0.0
- True
- >>> RandomRDDs.uniformRDD(sc, 100, 4).getNumPartitions()
- 4
- >>> parts = RandomRDDs.uniformRDD(sc, 100, seed=4).getNumPartitions()
- >>> parts == sc.defaultParallelism
- True
- """
- jrdd = sc._jvm.PythonMLLibAPI().uniformRDD(sc._jsc, size, numPartitions, seed)
- uniform = RDD(jrdd, sc, NoOpSerializer())
- return uniform.map(lambda bytes: _deserialize_double(bytearray(bytes)))
-
- @staticmethod
- def normalRDD(sc, size, numPartitions=None, seed=None):
- """
- Generates an RDD comprised of i.i.d. samples from the standard normal
- distribution.
-
- To transform the distribution in the generated RDD from standard normal
- to some other normal N(mean, sigma^2), use
- C{RandomRDDs.normal(sc, n, p, seed)\
- .map(lambda v: mean + sigma * v)}
-
- >>> x = RandomRDDs.normalRDD(sc, 1000, seed=1L)
- >>> stats = x.stats()
- >>> stats.count()
- 1000L
- >>> abs(stats.mean() - 0.0) < 0.1
- True
- >>> abs(stats.stdev() - 1.0) < 0.1
- True
- """
- jrdd = sc._jvm.PythonMLLibAPI().normalRDD(sc._jsc, size, numPartitions, seed)
- normal = RDD(jrdd, sc, NoOpSerializer())
- return normal.map(lambda bytes: _deserialize_double(bytearray(bytes)))
-
- @staticmethod
- def poissonRDD(sc, mean, size, numPartitions=None, seed=None):
- """
- Generates an RDD comprised of i.i.d. samples from the Poisson
- distribution with the input mean.
-
- >>> mean = 100.0
- >>> x = RandomRDDs.poissonRDD(sc, mean, 1000, seed=1L)
- >>> stats = x.stats()
- >>> stats.count()
- 1000L
- >>> abs(stats.mean() - mean) < 0.5
- True
- >>> from math import sqrt
- >>> abs(stats.stdev() - sqrt(mean)) < 0.5
- True
- """
- jrdd = sc._jvm.PythonMLLibAPI().poissonRDD(sc._jsc, mean, size, numPartitions, seed)
- poisson = RDD(jrdd, sc, NoOpSerializer())
- return poisson.map(lambda bytes: _deserialize_double(bytearray(bytes)))
-
- @staticmethod
- def uniformVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
- """
- Generates an RDD comprised of vectors containing i.i.d. samples drawn
- from the uniform distribution U(0.0, 1.0).
-
- >>> import numpy as np
- >>> mat = np.matrix(RandomRDDs.uniformVectorRDD(sc, 10, 10).collect())
- >>> mat.shape
- (10, 10)
- >>> mat.max() <= 1.0 and mat.min() >= 0.0
- True
- >>> RandomRDDs.uniformVectorRDD(sc, 10, 10, 4).getNumPartitions()
- 4
- """
- jrdd = sc._jvm.PythonMLLibAPI() \
- .uniformVectorRDD(sc._jsc, numRows, numCols, numPartitions, seed)
- uniform = RDD(jrdd, sc, NoOpSerializer())
- return uniform.map(lambda bytes: _deserialize_double_vector(bytearray(bytes)))
-
- @staticmethod
- def normalVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
- """
- Generates an RDD comprised of vectors containing i.i.d. samples drawn
- from the standard normal distribution.
-
- >>> import numpy as np
- >>> mat = np.matrix(RandomRDDs.normalVectorRDD(sc, 100, 100, seed=1L).collect())
- >>> mat.shape
- (100, 100)
- >>> abs(mat.mean() - 0.0) < 0.1
- True
- >>> abs(mat.std() - 1.0) < 0.1
- True
- """
- jrdd = sc._jvm.PythonMLLibAPI() \
- .normalVectorRDD(sc._jsc, numRows, numCols, numPartitions, seed)
- normal = RDD(jrdd, sc, NoOpSerializer())
- return normal.map(lambda bytes: _deserialize_double_vector(bytearray(bytes)))
-
- @staticmethod
- def poissonVectorRDD(sc, mean, numRows, numCols, numPartitions=None, seed=None):
- """
- Generates an RDD comprised of vectors containing i.i.d. samples drawn
- from the Poisson distribution with the input mean.
-
- >>> import numpy as np
- >>> mean = 100.0
- >>> rdd = RandomRDDs.poissonVectorRDD(sc, mean, 100, 100, seed=1L)
- >>> mat = np.mat(rdd.collect())
- >>> mat.shape
- (100, 100)
- >>> abs(mat.mean() - mean) < 0.5
- True
- >>> from math import sqrt
- >>> abs(mat.std() - sqrt(mean)) < 0.5
- True
- """
- jrdd = sc._jvm.PythonMLLibAPI() \
- .poissonVectorRDD(sc._jsc, mean, numRows, numCols, numPartitions, seed)
- poisson = RDD(jrdd, sc, NoOpSerializer())
- return poisson.map(lambda bytes: _deserialize_double_vector(bytearray(bytes)))
-
-
-def _test():
- import doctest
- from pyspark.context import SparkContext
- globs = globals().copy()
- # The small batch size here ensures that we see multiple batches,
- # even in these small test examples:
- globs['sc'] = SparkContext('local[2]', 'PythonTest', batchSize=2)
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
- globs['sc'].stop()
- if failure_count:
- exit(-1)
-
-
-if __name__ == "__main__":
- _test()
http://git-wip-us.apache.org/repos/asf/spark/blob/c98dc0e8/python/run-tests
----------------------------------------------------------------------
diff --git a/python/run-tests b/python/run-tests
index d671da4..51df52d 100755
--- a/python/run-tests
+++ b/python/run-tests
@@ -73,7 +73,7 @@ run_test "pyspark/mllib/_common.py"
run_test "pyspark/mllib/classification.py"
run_test "pyspark/mllib/clustering.py"
run_test "pyspark/mllib/linalg.py"
-run_test "pyspark/mllib/random.py"
+run_test "pyspark/mllib/rand.py"
run_test "pyspark/mllib/recommendation.py"
run_test "pyspark/mllib/regression.py"
run_test "pyspark/mllib/stat.py"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org