You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2024/08/19 14:41:24 UTC
(beam) branch master updated: Add hard delete function to multi_process_shared (#32238)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new aa13a692cd8 Add hard delete function to multi_process_shared (#32238)
aa13a692cd8 is described below
commit aa13a692cd8dd2ff75821708bde523df4c0a247b
Author: Danny McCormick <da...@google.com>
AuthorDate: Mon Aug 19 16:41:17 2024 +0200
Add hard delete function to multi_process_shared (#32238)
* Add hard delete function to multi_process_shared
* better testing
---
.../apache_beam/utils/multi_process_shared.py | 23 ++++++++++++
.../apache_beam/utils/multi_process_shared_test.py | 43 ++++++++++++++++++++++
2 files changed, 66 insertions(+)
diff --git a/sdks/python/apache_beam/utils/multi_process_shared.py b/sdks/python/apache_beam/utils/multi_process_shared.py
index 0c901500348..aecb1284a1d 100644
--- a/sdks/python/apache_beam/utils/multi_process_shared.py
+++ b/sdks/python/apache_beam/utils/multi_process_shared.py
@@ -136,6 +136,12 @@ class _SingletonEntry:
del self.obj
self.initialied = False
+ def unsafe_hard_delete(self):
+ with self.lock:
+ if self.initialied:
+ del self.obj
+ self.initialied = False
+
class _SingletonManager:
entries: Dict[Any, Any] = {}
@@ -153,6 +159,9 @@ class _SingletonManager:
def release_singleton(self, tag, obj):
return self.entries[tag].release(obj)
+ def unsafe_hard_delete_singleton(self, tag):
+ return self.entries[tag].unsafe_hard_delete()
+
_process_level_singleton_manager = _SingletonManager()
@@ -169,6 +178,9 @@ _SingletonRegistrar.register(
_SingletonRegistrar.register(
'release_singleton',
callable=_process_level_singleton_manager.release_singleton)
+_SingletonRegistrar.register(
+ 'unsafe_hard_delete_singleton',
+ callable=_process_level_singleton_manager.unsafe_hard_delete_singleton)
# By default, objects registered with BaseManager.register will have only
@@ -294,6 +306,17 @@ class MultiProcessShared(Generic[T]):
def release(self, obj):
self._manager.release_singleton(self._tag, obj.get_auto_proxy_object())
+ def unsafe_hard_delete(self):
+ """Force deletes the underlying object
+
+ This function should be used with great care since any other references
+ to this object will now be invalid and may lead to strange errors. Only
+ call unsafe_hard_delete if either (a) you are sure no other references
+ to this object exist, or (b) you are ok with all existing references to
+ this object throwing strange errors when derefrenced.
+ """
+ self._get_manager().unsafe_hard_delete_singleton(self._tag)
+
def _create_server(self, address_file):
# We need to be able to authenticate with both the manager and the process.
self._serving_manager = _SingletonRegistrar(
diff --git a/sdks/python/apache_beam/utils/multi_process_shared_test.py b/sdks/python/apache_beam/utils/multi_process_shared_test.py
index 6654ec3799e..0b795763236 100644
--- a/sdks/python/apache_beam/utils/multi_process_shared_test.py
+++ b/sdks/python/apache_beam/utils/multi_process_shared_test.py
@@ -167,6 +167,49 @@ class MultiProcessSharedTest(unittest.TestCase):
with self.assertRaisesRegex(Exception, 'released'):
counter1.get()
+ def test_unsafe_hard_delete(self):
+ shared1 = multi_process_shared.MultiProcessShared(
+ Counter, tag='test_unsafe_hard_delete', always_proxy=True)
+ shared2 = multi_process_shared.MultiProcessShared(
+ Counter, tag='test_unsafe_hard_delete', always_proxy=True)
+
+ counter1 = shared1.acquire()
+ counter2 = shared2.acquire()
+ self.assertEqual(counter1.increment(), 1)
+ self.assertEqual(counter2.increment(), 2)
+
+ multi_process_shared.MultiProcessShared(
+ Counter, tag='test_unsafe_hard_delete').unsafe_hard_delete()
+
+ with self.assertRaises(Exception):
+ counter1.get()
+ with self.assertRaises(Exception):
+ counter2.get()
+
+ shared3 = multi_process_shared.MultiProcessShared(
+ Counter, tag='test_unsafe_hard_delete', always_proxy=True)
+
+ counter3 = shared3.acquire()
+
+ self.assertEqual(counter3.increment(), 1)
+
+ def test_unsafe_hard_delete_no_op(self):
+ shared1 = multi_process_shared.MultiProcessShared(
+ Counter, tag='test_unsafe_hard_delete_no_op', always_proxy=True)
+ shared2 = multi_process_shared.MultiProcessShared(
+ Counter, tag='test_unsafe_hard_delete_no_op', always_proxy=True)
+
+ counter1 = shared1.acquire()
+ counter2 = shared2.acquire()
+ self.assertEqual(counter1.increment(), 1)
+ self.assertEqual(counter2.increment(), 2)
+
+ multi_process_shared.MultiProcessShared(
+ Counter, tag='no_tag_to_delete').unsafe_hard_delete()
+
+ self.assertEqual(counter1.increment(), 3)
+ self.assertEqual(counter2.increment(), 4)
+
def test_release_always_proxy(self):
shared1 = multi_process_shared.MultiProcessShared(
Counter, tag='test_release_always_proxy', always_proxy=True)