You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/05/08 13:23:02 UTC
spark git commit: [SPARK-24131][PYSPARK][FOLLOWUP] Add
majorMinorVersion API to PySpark for determining Spark versions
Repository: spark
Updated Branches:
refs/heads/master e17567ca7 -> b54bbe57b
[SPARK-24131][PYSPARK][FOLLOWUP] Add majorMinorVersion API to PySpark for determining Spark versions
## What changes were proposed in this pull request?
More close to Scala API behavior when can't parse input by throwing exception. Add tests.
## How was this patch tested?
Added tests.
Author: Liang-Chi Hsieh <vi...@gmail.com>
Closes #21211 from viirya/SPARK-24131-followup.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b54bbe57
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b54bbe57
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b54bbe57
Branch: refs/heads/master
Commit: b54bbe57b33b00063596cd9588fa2461745ed571
Parents: e17567c
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Tue May 8 21:22:54 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Tue May 8 21:22:54 2018 +0800
----------------------------------------------------------------------
python/pyspark/tests.py | 4 ++++
python/pyspark/util.py | 37 ++++++++++++++++++++++---------------
2 files changed, 26 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b54bbe57/python/pyspark/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py
index 7b8ce2c..498d6b5 100644
--- a/python/pyspark/tests.py
+++ b/python/pyspark/tests.py
@@ -2312,6 +2312,10 @@ class UtilTests(PySparkTestCase):
self.assertTrue('NullPointerException' in _exception_message(context.exception))
+ def test_parsing_version_string(self):
+ from pyspark.util import VersionUtils
+ self.assertRaises(ValueError, lambda: VersionUtils.majorMinorVersion("abced"))
+
@unittest.skipIf(not _have_scipy, "SciPy not installed")
class SciPyTests(PySparkTestCase):
http://git-wip-us.apache.org/repos/asf/spark/blob/b54bbe57/python/pyspark/util.py
----------------------------------------------------------------------
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index 04df835..59cc2a6 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -62,24 +62,31 @@ def _get_argspec(f):
return argspec
-def majorMinorVersion(version):
+class VersionUtils(object):
"""
- Get major and minor version numbers for given Spark version string.
-
- >>> version = "2.4.0"
- >>> majorMinorVersion(version)
- (2, 4)
+ Provides utility method to determine Spark versions with given input string.
+ """
+ @staticmethod
+ def majorMinorVersion(sparkVersion):
+ """
+ Given a Spark version string, return the (major version number, minor version number).
+ E.g., for 2.0.1-SNAPSHOT, return (2, 0).
- >>> version = "abc"
- >>> majorMinorVersion(version) is None
- True
+ >>> sparkVersion = "2.4.0"
+ >>> VersionUtils.majorMinorVersion(sparkVersion)
+ (2, 4)
+ >>> sparkVersion = "2.3.0-SNAPSHOT"
+ >>> VersionUtils.majorMinorVersion(sparkVersion)
+ (2, 3)
- """
- m = re.search('^(\d+)\.(\d+)(\..*)?$', version)
- if m is None:
- return None
- else:
- return (int(m.group(1)), int(m.group(2)))
+ """
+ m = re.search('^(\d+)\.(\d+)(\..*)?$', sparkVersion)
+ if m is not None:
+ return (int(m.group(1)), int(m.group(2)))
+ else:
+ raise ValueError("Spark tried to parse '%s' as a Spark" % sparkVersion +
+ " version string, but it could not find the major and minor" +
+ " version numbers.")
if __name__ == "__main__":
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org