You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/07/11 22:59:44 UTC
[1/2] incubator-beam git commit: Pickle only used symbols from
__main__ namespace.
Repository: incubator-beam
Updated Branches:
refs/heads/apache_beam [created] cd3bcd51f
Pickle only used symbols from __main__ namespace.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2809edc5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2809edc5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2809edc5
Branch: refs/heads/apache_beam
Commit: 2809edc5c05701c2d97d93cd355337bc86a368e3
Parents: 73168b2
Author: Robert Bradshaw <ro...@google.com>
Authored: Mon Jul 11 14:06:39 2016 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Mon Jul 11 15:59:07 2016 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/internal/pickler.py | 18 ++++++++++++++++++
sdks/python/apache_beam/internal/pickler_test.py | 3 +++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2809edc5/sdks/python/apache_beam/internal/pickler.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/pickler.py b/sdks/python/apache_beam/internal/pickler.py
index 898e04b..f427aa5 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -159,6 +159,24 @@ if 'save_module' in dir(dill.dill):
return old_save_module_dict(pickler, obj)
dill.dill.save_module_dict = new_save_module_dict
+
+ old_save_function = dill.dill.save_function
+
+ @dill.dill.register(types.FunctionType)
+ def new_save_function(pickler, obj):
+ globs = obj.__globals__ if dill.dill.PY3 else obj.func_globals
+ if (dill.dill.is_dill(pickler) and globs == pickler._main.__dict__
+ and not pickler._recurse):
+ try:
+ pickler._recurse = True
+ return old_save_function(pickler, obj)
+ finally:
+ pickler._recurse = False
+ else:
+ return old_save_function(pickler, obj)
+ dill.dill.save_function = new_save_function
+
+
def _nest_dill_logging():
"""Prefix all dill logging with its depth in the callstack.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2809edc5/sdks/python/apache_beam/internal/pickler_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/pickler_test.py b/sdks/python/apache_beam/internal/pickler_test.py
index 20de923..0952a26 100644
--- a/sdks/python/apache_beam/internal/pickler_test.py
+++ b/sdks/python/apache_beam/internal/pickler_test.py
@@ -40,6 +40,9 @@ class PicklerTest(unittest.TestCase):
['abc', 'def'],
loads(dumps(module_test.get_lambda_with_globals()))('abc def'))
+ def test_lambda_with_main_globals(self):
+ self.assertEquals(unittest, loads(dumps(lambda: unittest))())
+
def test_lambda_with_closure(self):
"""Tests that the closure of a function is preserved."""
self.assertEquals(
[2/2] incubator-beam git commit: Closes #629
Posted by ro...@apache.org.
Closes #629
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/cd3bcd51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/cd3bcd51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/cd3bcd51
Branch: refs/heads/apache_beam
Commit: cd3bcd51fa83e19b7d284d6394da98d1462c5732
Parents: 73168b2 2809edc
Author: Robert Bradshaw <ro...@google.com>
Authored: Mon Jul 11 15:59:15 2016 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Mon Jul 11 15:59:15 2016 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/internal/pickler.py | 18 ++++++++++++++++++
sdks/python/apache_beam/internal/pickler_test.py | 3 +++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------