You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/07/11 22:59:44 UTC

[1/2] incubator-beam git commit: Pickle only used symbols from __main__ namespace.

Repository: incubator-beam
Updated Branches:
  refs/heads/apache_beam [created] cd3bcd51f


Pickle only used symbols from __main__ namespace.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2809edc5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2809edc5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2809edc5

Branch: refs/heads/apache_beam
Commit: 2809edc5c05701c2d97d93cd355337bc86a368e3
Parents: 73168b2
Author: Robert Bradshaw <ro...@google.com>
Authored: Mon Jul 11 14:06:39 2016 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Mon Jul 11 15:59:07 2016 -0700

----------------------------------------------------------------------
 sdks/python/apache_beam/internal/pickler.py      | 18 ++++++++++++++++++
 sdks/python/apache_beam/internal/pickler_test.py |  3 +++
 2 files changed, 21 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2809edc5/sdks/python/apache_beam/internal/pickler.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/pickler.py b/sdks/python/apache_beam/internal/pickler.py
index 898e04b..f427aa5 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -159,6 +159,24 @@ if 'save_module' in dir(dill.dill):
       return old_save_module_dict(pickler, obj)
   dill.dill.save_module_dict = new_save_module_dict
 
+
+  old_save_function = dill.dill.save_function
+
+  @dill.dill.register(types.FunctionType)
+  def new_save_function(pickler, obj):
+    globs = obj.__globals__ if dill.dill.PY3 else obj.func_globals
+    if (dill.dill.is_dill(pickler) and globs == pickler._main.__dict__
+        and not pickler._recurse):
+      try:
+        pickler._recurse = True
+        return old_save_function(pickler, obj)
+      finally:
+        pickler._recurse = False
+    else:
+      return old_save_function(pickler, obj)
+  dill.dill.save_function = new_save_function
+
+
   def _nest_dill_logging():
     """Prefix all dill logging with its depth in the callstack.
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2809edc5/sdks/python/apache_beam/internal/pickler_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/pickler_test.py b/sdks/python/apache_beam/internal/pickler_test.py
index 20de923..0952a26 100644
--- a/sdks/python/apache_beam/internal/pickler_test.py
+++ b/sdks/python/apache_beam/internal/pickler_test.py
@@ -40,6 +40,9 @@ class PicklerTest(unittest.TestCase):
         ['abc', 'def'],
         loads(dumps(module_test.get_lambda_with_globals()))('abc def'))
 
+  def test_lambda_with_main_globals(self):
+    self.assertEquals(unittest, loads(dumps(lambda: unittest))())
+
   def test_lambda_with_closure(self):
     """Tests that the closure of a function is preserved."""
     self.assertEquals(


[2/2] incubator-beam git commit: Closes #629

Posted by ro...@apache.org.
Closes #629


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/cd3bcd51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/cd3bcd51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/cd3bcd51

Branch: refs/heads/apache_beam
Commit: cd3bcd51fa83e19b7d284d6394da98d1462c5732
Parents: 73168b2 2809edc
Author: Robert Bradshaw <ro...@google.com>
Authored: Mon Jul 11 15:59:15 2016 -0700
Committer: Robert Bradshaw <ro...@google.com>
Committed: Mon Jul 11 15:59:15 2016 -0700

----------------------------------------------------------------------
 sdks/python/apache_beam/internal/pickler.py      | 18 ++++++++++++++++++
 sdks/python/apache_beam/internal/pickler_test.py |  3 +++
 2 files changed, 21 insertions(+)
----------------------------------------------------------------------