You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "nalin garg (JIRA)" <ji...@apache.org> on 2016/05/31 19:29:12 UTC
[jira] [Commented] (SPARK-15679) Passing functions do not work in
classes
[ https://issues.apache.org/jira/browse/SPARK-15679?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15308425#comment-15308425 ]
nalin garg commented on SPARK-15679:
------------------------------------
[~sowen] below is stack trace:
Traceback (most recent call last):
File "/Users/nalingarg/PycharmProjects/parallel_check/parallel_check.py", line 90, in <module>
r.count()
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py", line 1004, in count
return self.mapPartitions(lambda i: [sum(1 for _ in i)]).sum()
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py", line 995, in sum
return self.mapPartitions(lambda x: [sum(x)]).fold(0, operator.add)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py", line 869, in fold
vals = self.mapPartitions(func).collect()
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py", line 771, in collect
port = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py", line 2379, in _jrdd
pickled_cmd, bvars, env, includes = _prepare_for_python_RDD(self.ctx, command, self)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/rdd.py", line 2299, in _prepare_for_python_RDD
pickled_command = ser.dumps(command)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/serializers.py", line 428, in dumps
return cloudpickle.dumps(obj, 2)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 646, in dumps
cp.dump(obj)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 107, in dump
return Pickler.dump(self, obj)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 224, in dump
self.save(obj)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 568, in save_tuple
save(element)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 199, in save_function
self.save_function_tuple(obj)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 236, in save_function_tuple
save((code, closure, base_globals))
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 554, in save_tuple
save(element)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 606, in save_list
self._batch_appends(iter(obj))
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 639, in _batch_appends
save(x)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 199, in save_function
self.save_function_tuple(obj)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 236, in save_function_tuple
save((code, closure, base_globals))
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 554, in save_tuple
save(element)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 606, in save_list
self._batch_appends(iter(obj))
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 639, in _batch_appends
save(x)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 199, in save_function
self.save_function_tuple(obj)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 236, in save_function_tuple
save((code, closure, base_globals))
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 554, in save_tuple
save(element)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 606, in save_list
self._batch_appends(iter(obj))
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 639, in _batch_appends
save(x)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 199, in save_function
self.save_function_tuple(obj)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 236, in save_function_tuple
save((code, closure, base_globals))
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 554, in save_tuple
save(element)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 606, in save_list
self._batch_appends(iter(obj))
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 642, in _batch_appends
save(tmp[0])
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 390, in save_instancemethod
obj=obj)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 524, in save_reduce
save(args)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 554, in save_tuple
save(element)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 509, in save_reduce
save(cls)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 374, in save_global
self.save(dd)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 687, in _batch_setitems
save(v)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 193, in save_function
self.save_function_tuple(obj)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 241, in save_function_tuple
save(f_globals)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 692, in _batch_setitems
save(v)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "/Users/nalingarg/Documents/spark-1.6.1-bin-hadoop2.6/python/pyspark/cloudpickle.py", line 542, in save_reduce
save(state)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 687, in _batch_setitems
save(v)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 306, in save
rv = reduce(self.proto)
TypeError: 'JavaPackage' object is not callable
> Passing functions do not work in classes
> ----------------------------------------
>
> Key: SPARK-15679
> URL: https://issues.apache.org/jira/browse/SPARK-15679
> Project: Spark
> Issue Type: Bug
> Reporter: nalin garg
>
> {code}
> from pyspark import SparkContext
> from pyspark.sql import SQLContext, HiveContext
> import csv, io, StringIO
> from pyspark.sql.functions import *
> from pyspark.sql import Row
> from pyspark.sql import *
> from pyspark.sql import functions as F
> from pyspark.sql.functions import asc, desc
> sc = SparkContext("local", "Summary Report")
> sqlContext = SQLContext(sc)
> class ParallelBuild(object):
> def myFunc(self, s):
> l = s.split(',')
> print l[0], l[1]
> return l[0]
> def list_to_csv_str(x):
> output = StringIO.StringIO("")
> csv.writer(output).writerow(x)
> return output.getvalue().strip()
> def run(self):
> data = [(1,'a'), (1,'b'), (1,'c'), (2,'d'), (3,'r'), (4,'a'), (2,'t'), (3,'y'), (1,'f')]
> df = sqlContext.createDataFrame(data, schema= ['uid', 'address_uid'])
> return df
> if __name__ == "__main__":
> df_rdd = ParallelBuild().run().map(lambda line: line).persist()
> r = df_rdd.map(ParallelBuild().myFunc)
> r.count()
> {code}
> Above code returns JavaPackage not callable exception.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org