You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2024/03/19 18:25:21 UTC
(beam) branch users/damccorm/critSection created (now eda0b519d49)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a change to branch users/damccorm/critSection
in repository https://gitbox.apache.org/repos/asf/beam.git
at eda0b519d49 Minimize scope of expensive lock
This branch includes the following new commits:
new eda0b519d49 Minimize scope of expensive lock
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(beam) 01/01: Minimize scope of expensive lock
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch users/damccorm/critSection
in repository https://gitbox.apache.org/repos/asf/beam.git
commit eda0b519d49c24cbb148eebbf34494dba1adaaa4
Author: Danny McCormick <da...@google.com>
AuthorDate: Tue Mar 19 14:25:14 2024 -0400
Minimize scope of expensive lock
---
sdks/python/apache_beam/runners/worker/sdk_worker.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py
index 04b58a6f198..8bb367d89e4 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py
@@ -607,12 +607,16 @@ class BundleProcessorCache(object):
# type: () -> None
def shutdown_inactive_bundle_processors():
# type: () -> None
+ access_times = []
with self._lock:
- for descriptor_id, last_access_time in self.last_access_times.items():
- if (time.time() - last_access_time >
- DEFAULT_BUNDLE_PROCESSOR_CACHE_SHUTDOWN_THRESHOLD_S):
- BundleProcessorCache._shutdown_cached_bundle_processors(
- self.cached_bundle_processors[descriptor_id])
+ # Shutdown can be expensive, keep out of lock
+ access_times = list(self.last_access_times.items())
+
+ for descriptor_id, last_access_time in access_times:
+ if (time.time() - last_access_time >
+ DEFAULT_BUNDLE_PROCESSOR_CACHE_SHUTDOWN_THRESHOLD_S):
+ BundleProcessorCache._shutdown_cached_bundle_processors(
+ self.cached_bundle_processors[descriptor_id])
self.periodic_shutdown = PeriodicThread(
DEFAULT_BUNDLE_PROCESSOR_CACHE_SHUTDOWN_THRESHOLD_S,