You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by "Sourabh Bajaj (JIRA)" <ji...@apache.org> on 2017/01/31 18:44:51 UTC
[jira] (BEAM-804) Python Pipeline Option save_main_session
non-functional
[ https://issues.apache.org/jira/browse/BEAM-804?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15847279#comment-15847279 ]
Sourabh Bajaj commented on BEAM-804:
------------------------------------
This should be fixed by https://github.com/apache/beam/commit/aef4858b80dfacf3401e6672b9373c82a8e77027
> Python Pipeline Option save_main_session non-functional
> -------------------------------------------------------
>
> Key: BEAM-804
> URL: https://issues.apache.org/jira/browse/BEAM-804
> Project: Beam
> Issue Type: Bug
> Components: sdk-py
> Environment: OSX El Capitan, google-cloud-dataflow==0.4.3, python 2.7.12
> Reporter: Zoran Bujila
> Priority: Critical
>
> When trying to use the option --save_main_session a pickling error occurs.
> pickle.PicklingError: Can't pickle <class 'apache_beam.internal.clients.dataflow.dataflow_v1b3_messages.TypeValueValuesEnum'>: it's not found as apache_beam.internal.clients.dataflow.dataflow_v1b3_messages.TypeValueValuesEnu
> This prevents the use of this option which is desirable as there is an expensive object that needs to be created on each worker in my pipeline and I would like to have this object created only once per worker. It is not practical to have it inline with the ParDo function unless I make the batch size sent to the ParDo quite large. Doing this seems to lead to idle workers and I would ideally want to bring the batch size way down.
> The "Affects Version" option above doesn't have a 0.4.3 version in the drop down so I did not populate it. However, this was a problem with 0.4.1 and has not been corrected with 0.4.3.
> I don't see where I can attach a file, so here is the entire error.
> 2016-10-24 10:00:16,071 <module> The oauth2client.contrib.multistore_file module has been deprecated and will be removed in the next release of oauth2client. Please migrate to multiprocess_file_storage.
> 2016-10-24 10:00:16,127 __init__ Direct usage of TextFileSink is deprecated. Please use 'textio.WriteToText()' instead of directly instantiating a TextFileSink object.
> Traceback (most recent call last):
> File "00test.py", line 41, in <module>
> p.run()
> File "/usr/local/lib/python2.7/site-packages/apache_beam/pipeline.py", line 159, in run
> return self.runner.run(self)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/dataflow_runner.py", line 172, in run
> self.dataflow_client.create_job(self.job))
> File "/usr/local/lib/python2.7/site-packages/apache_beam/utils/retry.py", line 160, in wrapper
> return fun(*args, **kwargs)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/internal/apiclient.py", line 375, in create_job
> job.options, file_copy=self._gcs_file_copy)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/utils/dependency.py", line 325, in stage_job_resources
> pickler.dump_session(pickled_session_file)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 204, in dump_session
> return dill.dump_session(file_path)
> File "/usr/local/lib/python2.7/site-packages/dill/dill.py", line 333, in dump_session
> pickler.dump(main)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 224, in dump
> self.save(obj)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
> f(self, obj) # Call unbound method with explicit self
> File "/usr/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 123, in save_module
> return old_save_module(pickler, obj)
> File "/usr/local/lib/python2.7/site-packages/dill/dill.py", line 1168, in save_module
> state=_main_dict)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 425, in save_reduce
> save(state)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
> f(self, obj) # Call unbound method with explicit self
> File "/usr/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 159, in new_save_module_dict
> return old_save_module_dict(pickler, obj)
> File "/usr/local/lib/python2.7/site-packages/dill/dill.py", line 835, in save_module_dict
> StockPickler.save_dict(pickler, obj)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 655, in save_dict
> self._batch_setitems(obj.iteritems())
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 687, in _batch_setitems
> save(v)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 331, in save
> self.save_reduce(obj=obj, *rv)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 425, in save_reduce
> save(state)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
> f(self, obj) # Call unbound method with explicit self
> File "/usr/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 159, in new_save_module_dict
> return old_save_module_dict(pickler, obj)
> File "/usr/local/lib/python2.7/site-packages/dill/dill.py", line 835, in save_module_dict
> StockPickler.save_dict(pickler, obj)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 655, in save_dict
> self._batch_setitems(obj.iteritems())
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 687, in _batch_setitems
> save(v)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 331, in save
> self.save_reduce(obj=obj, *rv)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 425, in save_reduce
> save(state)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
> f(self, obj) # Call unbound method with explicit self
> File "/usr/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 159, in new_save_module_dict
> return old_save_module_dict(pickler, obj)
> File "/usr/local/lib/python2.7/site-packages/dill/dill.py", line 835, in save_module_dict
> StockPickler.save_dict(pickler, obj)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 655, in save_dict
> self._batch_setitems(obj.iteritems())
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 687, in _batch_setitems
> save(v)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 331, in save
> self.save_reduce(obj=obj, *rv)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 425, in save_reduce
> save(state)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
> f(self, obj) # Call unbound method with explicit self
> File "/usr/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 159, in new_save_module_dict
> return old_save_module_dict(pickler, obj)
> File "/usr/local/lib/python2.7/site-packages/dill/dill.py", line 835, in save_module_dict
> StockPickler.save_dict(pickler, obj)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 655, in save_dict
> self._batch_setitems(obj.iteritems())
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 687, in _batch_setitems
> save(v)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 331, in save
> self.save_reduce(obj=obj, *rv)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 425, in save_reduce
> save(state)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
> f(self, obj) # Call unbound method with explicit self
> File "/usr/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 159, in new_save_module_dict
> return old_save_module_dict(pickler, obj)
> File "/usr/local/lib/python2.7/site-packages/dill/dill.py", line 835, in save_module_dict
> StockPickler.save_dict(pickler, obj)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 655, in save_dict
> self._batch_setitems(obj.iteritems())
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 687, in _batch_setitems
> save(v)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
> f(self, obj) # Call unbound method with explicit self
> File "/usr/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 159, in new_save_module_dict
> return old_save_module_dict(pickler, obj)
> File "/usr/local/lib/python2.7/site-packages/dill/dill.py", line 835, in save_module_dict
> StockPickler.save_dict(pickler, obj)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 655, in save_dict
> self._batch_setitems(obj.iteritems())
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 687, in _batch_setitems
> save(v)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 331, in save
> self.save_reduce(obj=obj, *rv)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 400, in save_reduce
> save(func)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 286, in save
> f(self, obj) # Call unbound method with explicit self
> File "/usr/local/lib/python2.7/site-packages/dill/dill.py", line 1231, in save_type
> StockPickler.save_global(pickler, obj)
> File "/usr/local/Cellar/python/2.7.12_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pickle.py", line 754, in save_global
> (obj, module, name))
> pickle.PicklingError: Can't pickle <class 'apache_beam.internal.clients.dataflow.dataflow_v1b3_messages.TypeValueValuesEnum'>: it's not found as apache_beam.internal.clients.dataflow.dataflow_v1b3_messages.TypeValueValuesEnum
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)