You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bahir.apache.org by lr...@apache.org on 2016/06/10 15:24:22 UTC
[43/50] [abbrv] bahir git commit: [SPARK-13848][SPARK-5185] Update to
Py4J 0.9.2 in order to fix classloading issue
[SPARK-13848][SPARK-5185] Update to Py4J 0.9.2 in order to fix classloading issue
This patch upgrades Py4J from 0.9.1 to 0.9.2 in order to include a patch which modifies Py4J to use the current thread's ContextClassLoader when performing reflection / class loading. This is necessary in order to fix [SPARK-5185](https://issues.apache.org/jira/browse/SPARK-5185), a longstanding issue affecting the use of `--jars` and `--packages` in PySpark.
In order to demonstrate that the fix works, I removed the workarounds which were added as part of [SPARK-6027](https://issues.apache.org/jira/browse/SPARK-6027) / #4779 and other patches.
Py4J diff: https://github.com/bartdag/py4j/compare/0.9.1...0.9.2
/cc zsxwing tdas davies brkyvz
Author: Josh Rosen <jo...@databricks.com>
Closes #11687 from JoshRosen/py4j-0.9.2.
Project: http://git-wip-us.apache.org/repos/asf/bahir/repo
Commit: http://git-wip-us.apache.org/repos/asf/bahir/commit/54040f37
Tree: http://git-wip-us.apache.org/repos/asf/bahir/tree/54040f37
Diff: http://git-wip-us.apache.org/repos/asf/bahir/diff/54040f37
Branch: refs/heads/master
Commit: 54040f372efee60b3283fbd15d030b5e8d3aba87
Parents: a8dc23a
Author: Josh Rosen <jo...@databricks.com>
Authored: Mon Mar 14 12:22:02 2016 -0700
Committer: Josh Rosen <jo...@databricks.com>
Committed: Mon Mar 14 12:22:02 2016 -0700
----------------------------------------------------------------------
streaming-mqtt/python/mqtt.py | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bahir/blob/54040f37/streaming-mqtt/python/mqtt.py
----------------------------------------------------------------------
diff --git a/streaming-mqtt/python/mqtt.py b/streaming-mqtt/python/mqtt.py
index 388e952..8848a70 100644
--- a/streaming-mqtt/python/mqtt.py
+++ b/streaming-mqtt/python/mqtt.py
@@ -38,18 +38,15 @@ class MQTTUtils(object):
:param storageLevel: RDD storage level.
:return: A DStream object
"""
- jlevel = ssc._sc._getJavaStorageLevel(storageLevel)
-
try:
- helperClass = ssc._jvm.java.lang.Thread.currentThread().getContextClassLoader() \
- .loadClass("org.apache.spark.streaming.mqtt.MQTTUtilsPythonHelper")
- helper = helperClass.newInstance()
- jstream = helper.createStream(ssc._jssc, brokerUrl, topic, jlevel)
- except Py4JJavaError as e:
- if 'ClassNotFoundException' in str(e.java_exception):
+ helper = ssc._jvm.org.apache.spark.streaming.mqtt.MQTTUtilsPythonHelper()
+ except TypeError as e:
+ if str(e) == "'JavaPackage' object is not callable":
MQTTUtils._printErrorMsg(ssc.sparkContext)
raise
+ jlevel = ssc._sc._getJavaStorageLevel(storageLevel)
+ jstream = helper.createStream(ssc._jssc, brokerUrl, topic, jlevel)
return DStream(jstream, ssc, UTF8Deserializer())
@staticmethod