You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2015/02/17 21:05:15 UTC
spark git commit: [SPARK-5864] [PySpark] support .jar as python
package
Repository: spark
Updated Branches:
refs/heads/master 49c19fdba -> fc4eb9505
[SPARK-5864] [PySpark] support .jar as python package
A jar file containing Python sources in it could be used as a Python package, just like zip file.
spark-submit already put the jar file into PYTHONPATH, this patch also put it in the sys.path, then it could be used in Python worker.
Author: Davies Liu <da...@databricks.com>
Closes #4652 from davies/jar and squashes the following commits:
17d3f76 [Davies Liu] support .jar as python package
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fc4eb950
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fc4eb950
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fc4eb950
Branch: refs/heads/master
Commit: fc4eb9505adda192eb38cb4454d532027690bfa3
Parents: 49c19fd
Author: Davies Liu <da...@databricks.com>
Authored: Tue Feb 17 12:05:06 2015 -0800
Committer: Patrick Wendell <pa...@databricks.com>
Committed: Tue Feb 17 12:05:06 2015 -0800
----------------------------------------------------------------------
python/pyspark/context.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fc4eb950/python/pyspark/context.py
----------------------------------------------------------------------
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index bf1f61c..40b3152 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -64,6 +64,8 @@ class SparkContext(object):
_lock = Lock()
_python_includes = None # zip and egg files that need to be added to PYTHONPATH
+ PACKAGE_EXTENSIONS = ('.zip', '.egg', '.jar')
+
def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None,
environment=None, batchSize=0, serializer=PickleSerializer(), conf=None,
gateway=None, jsc=None, profiler_cls=BasicProfiler):
@@ -185,7 +187,7 @@ class SparkContext(object):
for path in self._conf.get("spark.submit.pyFiles", "").split(","):
if path != "":
(dirname, filename) = os.path.split(path)
- if filename.lower().endswith("zip") or filename.lower().endswith("egg"):
+ if filename[-4:].lower() in self.PACKAGE_EXTENSIONS:
self._python_includes.append(filename)
sys.path.insert(1, os.path.join(SparkFiles.getRootDirectory(), filename))
@@ -705,7 +707,7 @@ class SparkContext(object):
self.addFile(path)
(dirname, filename) = os.path.split(path) # dirname may be directory or HDFS/S3 prefix
- if filename.endswith('.zip') or filename.endswith('.ZIP') or filename.endswith('.egg'):
+ if filename[-4:].lower() in self.PACKAGE_EXTENSIONS:
self._python_includes.append(filename)
# for tests in local mode
sys.path.insert(1, os.path.join(SparkFiles.getRootDirectory(), filename))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org