You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2024/08/19 14:41:24 UTC

(beam) branch master updated: Add hard delete function to multi_process_shared (#32238)

This is an automated email from the ASF dual-hosted git repository.

damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new aa13a692cd8 Add hard delete function to multi_process_shared (#32238)
aa13a692cd8 is described below

commit aa13a692cd8dd2ff75821708bde523df4c0a247b
Author: Danny McCormick <da...@google.com>
AuthorDate: Mon Aug 19 16:41:17 2024 +0200

    Add hard delete function to multi_process_shared (#32238)
    
    * Add hard delete function to multi_process_shared
    
    * better testing
---
 .../apache_beam/utils/multi_process_shared.py      | 23 ++++++++++++
 .../apache_beam/utils/multi_process_shared_test.py | 43 ++++++++++++++++++++++
 2 files changed, 66 insertions(+)

diff --git a/sdks/python/apache_beam/utils/multi_process_shared.py b/sdks/python/apache_beam/utils/multi_process_shared.py
index 0c901500348..aecb1284a1d 100644
--- a/sdks/python/apache_beam/utils/multi_process_shared.py
+++ b/sdks/python/apache_beam/utils/multi_process_shared.py
@@ -136,6 +136,12 @@ class _SingletonEntry:
         del self.obj
         self.initialied = False
 
+  def unsafe_hard_delete(self):
+    with self.lock:
+      if self.initialied:
+        del self.obj
+        self.initialied = False
+
 
 class _SingletonManager:
   entries: Dict[Any, Any] = {}
@@ -153,6 +159,9 @@ class _SingletonManager:
   def release_singleton(self, tag, obj):
     return self.entries[tag].release(obj)
 
+  def unsafe_hard_delete_singleton(self, tag):
+    return self.entries[tag].unsafe_hard_delete()
+
 
 _process_level_singleton_manager = _SingletonManager()
 
@@ -169,6 +178,9 @@ _SingletonRegistrar.register(
 _SingletonRegistrar.register(
     'release_singleton',
     callable=_process_level_singleton_manager.release_singleton)
+_SingletonRegistrar.register(
+    'unsafe_hard_delete_singleton',
+    callable=_process_level_singleton_manager.unsafe_hard_delete_singleton)
 
 
 # By default, objects registered with BaseManager.register will have only
@@ -294,6 +306,17 @@ class MultiProcessShared(Generic[T]):
   def release(self, obj):
     self._manager.release_singleton(self._tag, obj.get_auto_proxy_object())
 
+  def unsafe_hard_delete(self):
+    """Force deletes the underlying object
+    
+      This function should be used with great care since any other references
+      to this object will now be invalid and may lead to strange errors. Only
+      call unsafe_hard_delete if either (a) you are sure no other references
+      to this object exist, or (b) you are ok with all existing references to
+      this object throwing strange errors when derefrenced.
+    """
+    self._get_manager().unsafe_hard_delete_singleton(self._tag)
+
   def _create_server(self, address_file):
     # We need to be able to authenticate with both the manager and the process.
     self._serving_manager = _SingletonRegistrar(
diff --git a/sdks/python/apache_beam/utils/multi_process_shared_test.py b/sdks/python/apache_beam/utils/multi_process_shared_test.py
index 6654ec3799e..0b795763236 100644
--- a/sdks/python/apache_beam/utils/multi_process_shared_test.py
+++ b/sdks/python/apache_beam/utils/multi_process_shared_test.py
@@ -167,6 +167,49 @@ class MultiProcessSharedTest(unittest.TestCase):
     with self.assertRaisesRegex(Exception, 'released'):
       counter1.get()
 
+  def test_unsafe_hard_delete(self):
+    shared1 = multi_process_shared.MultiProcessShared(
+        Counter, tag='test_unsafe_hard_delete', always_proxy=True)
+    shared2 = multi_process_shared.MultiProcessShared(
+        Counter, tag='test_unsafe_hard_delete', always_proxy=True)
+
+    counter1 = shared1.acquire()
+    counter2 = shared2.acquire()
+    self.assertEqual(counter1.increment(), 1)
+    self.assertEqual(counter2.increment(), 2)
+
+    multi_process_shared.MultiProcessShared(
+        Counter, tag='test_unsafe_hard_delete').unsafe_hard_delete()
+
+    with self.assertRaises(Exception):
+      counter1.get()
+    with self.assertRaises(Exception):
+      counter2.get()
+
+    shared3 = multi_process_shared.MultiProcessShared(
+        Counter, tag='test_unsafe_hard_delete', always_proxy=True)
+
+    counter3 = shared3.acquire()
+
+    self.assertEqual(counter3.increment(), 1)
+
+  def test_unsafe_hard_delete_no_op(self):
+    shared1 = multi_process_shared.MultiProcessShared(
+        Counter, tag='test_unsafe_hard_delete_no_op', always_proxy=True)
+    shared2 = multi_process_shared.MultiProcessShared(
+        Counter, tag='test_unsafe_hard_delete_no_op', always_proxy=True)
+
+    counter1 = shared1.acquire()
+    counter2 = shared2.acquire()
+    self.assertEqual(counter1.increment(), 1)
+    self.assertEqual(counter2.increment(), 2)
+
+    multi_process_shared.MultiProcessShared(
+        Counter, tag='no_tag_to_delete').unsafe_hard_delete()
+
+    self.assertEqual(counter1.increment(), 3)
+    self.assertEqual(counter2.increment(), 4)
+
   def test_release_always_proxy(self):
     shared1 = multi_process_shared.MultiProcessShared(
         Counter, tag='test_release_always_proxy', always_proxy=True)