You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ri...@apache.org on 2023/06/01 19:13:35 UTC
[beam] branch master updated: Make lulls message in Python SDK to be consistent with the implementation in Java SDK (#26936)
This is an automated email from the ASF dual-hosted git repository.
riteshghorse pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3dd239e2045 Make lulls message in Python SDK to be consistent with the implementation in Java SDK (#26936)
3dd239e2045 is described below
commit 3dd239e204593dd8fc67ceb4aaa558768fc3edbb
Author: bzablocki <bz...@google.com>
AuthorDate: Thu Jun 1 21:13:25 2023 +0200
Make lulls message in Python SDK to be consistent with the implementation in Java SDK (#26936)
* Make lulls message in Python SDK to be consistent with the implementation in Java SDK
* Adjust unit tests to match introduced changes
* Fix linting error
---
.../apache_beam/runners/worker/worker_status.py | 46 +++++++++++++---------
.../runners/worker/worker_status_test.py | 26 ++++++------
2 files changed, 42 insertions(+), 30 deletions(-)
diff --git a/sdks/python/apache_beam/runners/worker/worker_status.py b/sdks/python/apache_beam/runners/worker/worker_status.py
index 40f0d927c0e..a7f4890344a 100644
--- a/sdks/python/apache_beam/runners/worker/worker_status.py
+++ b/sdks/python/apache_beam/runners/worker/worker_status.py
@@ -97,7 +97,7 @@ def heap_dump():
def _state_cache_stats(state_cache):
- #type: (StateCache) -> str
+ # type: (StateCache) -> str
"""Gather state cache statistics."""
cache_stats = ['=' * 10 + ' CACHE STATS ' + '=' * 10]
@@ -236,34 +236,44 @@ class FnApiWorkerStatusHandler(object):
processor = bundle_process_cache.lookup(instruction)
if processor:
info = processor.state_sampler.get_info()
- self._log_lull_sampler_info(info)
+ self._log_lull_sampler_info(info, instruction)
- def _log_lull_sampler_info(self, sampler_info):
+ def _log_lull_sampler_info(self, sampler_info, instruction):
if not self._passed_lull_timeout_since_last_log():
return
if (sampler_info and sampler_info.time_since_transition and
sampler_info.time_since_transition > self.log_lull_timeout_ns):
+ lull_seconds = sampler_info.time_since_transition / 1e9
+
step_name = sampler_info.state_name.step_name
state_name = sampler_info.state_name.name
- lull_seconds = sampler_info.time_since_transition / 1e9
- state_lull_log = (
- 'Operation ongoing for over %.2f seconds in state %s' %
- (lull_seconds, state_name))
- step_name_log = (' in step %s ' % step_name) if step_name else ''
-
- exec_thread = getattr(sampler_info, 'tracked_thread', None)
- if exec_thread is not None:
- thread_frame = sys._current_frames().get(exec_thread.ident) # pylint: disable=protected-access
- stack_trace = '\n'.join(
- traceback.format_stack(thread_frame)) if thread_frame else ''
+ if step_name and state_name:
+ step_name_log = (
+ ' for PTransform{name=%s, state=%s}' % (step_name, state_name))
else:
- stack_trace = '-NOT AVAILABLE-'
+ step_name_log = ''
+
+ stack_trace = self._get_stack_trace(sampler_info)
_LOGGER.warning(
- '%s%s without returning. Current Traceback:\n%s',
- state_lull_log,
+ (
+ 'Operation ongoing in bundle %s%s for at least %.2f seconds'
+ ' without outputting or completing.\n'
+ 'Current Traceback:\n%s'),
+ instruction,
step_name_log,
- stack_trace)
+ lull_seconds,
+ stack_trace,
+ )
+
+ def _get_stack_trace(self, sampler_info):
+ exec_thread = getattr(sampler_info, 'tracked_thread', None)
+ if exec_thread is not None:
+ thread_frame = sys._current_frames().get(exec_thread.ident) # pylint: disable=protected-access
+ return '\n'.join(
+ traceback.format_stack(thread_frame)) if thread_frame else ''
+ else:
+ return '-NOT AVAILABLE-'
def _passed_lull_timeout_since_last_log(self) -> bool:
if (time.time() - self._last_lull_logged_secs >
diff --git a/sdks/python/apache_beam/runners/worker/worker_status_test.py b/sdks/python/apache_beam/runners/worker/worker_status_test.py
index 7da533991df..1004d21e7fd 100644
--- a/sdks/python/apache_beam/runners/worker/worker_status_test.py
+++ b/sdks/python/apache_beam/runners/worker/worker_status_test.py
@@ -88,7 +88,7 @@ class FnApiWorkerStatusHandlerTest(unittest.TestCase):
def test_log_lull_in_bundle_processor(self):
def get_state_sampler_info_for_lull(lull_duration_s):
- return statesampler.StateSamplerInfo(
+ return "bundle-id", statesampler.StateSamplerInfo(
CounterName('progress-msecs', 'stage_name', 'step_name'),
1,
lull_duration_s * 1e9,
@@ -98,31 +98,33 @@ class FnApiWorkerStatusHandlerTest(unittest.TestCase):
with mock.patch('logging.Logger.warning') as warn_mock:
with mock.patch('time.time') as time_mock:
time_mock.return_value = now
- sampler_info = get_state_sampler_info_for_lull(21 * 60)
- self.fn_status_handler._log_lull_sampler_info(sampler_info)
+ bundle_id, sampler_info = get_state_sampler_info_for_lull(21 * 60)
+ self.fn_status_handler._log_lull_sampler_info(sampler_info, bundle_id)
- processing_template = warn_mock.call_args[0][1]
+ bundle_id_template = warn_mock.call_args[0][1]
step_name_template = warn_mock.call_args[0][2]
- traceback = warn_mock.call_args = warn_mock.call_args[0][3]
+ processing_template = warn_mock.call_args[0][3]
+ traceback = warn_mock.call_args = warn_mock.call_args[0][4]
- self.assertIn('progress-msecs', processing_template)
+ self.assertIn('bundle-id', bundle_id_template)
self.assertIn('step_name', step_name_template)
+ self.assertEqual(21 * 60, processing_template)
self.assertIn('test_log_lull_in_bundle_processor', traceback)
with mock.patch('time.time') as time_mock:
time_mock.return_value = now + 6 * 60 # 6 minutes
- sampler_info = get_state_sampler_info_for_lull(21 * 60)
- self.fn_status_handler._log_lull_sampler_info(sampler_info)
+ bundle_id, sampler_info = get_state_sampler_info_for_lull(21 * 60)
+ self.fn_status_handler._log_lull_sampler_info(sampler_info, bundle_id)
with mock.patch('time.time') as time_mock:
time_mock.return_value = now + 21 * 60 # 21 minutes
- sampler_info = get_state_sampler_info_for_lull(10 * 60)
- self.fn_status_handler._log_lull_sampler_info(sampler_info)
+ bundle_id, sampler_info = get_state_sampler_info_for_lull(10 * 60)
+ self.fn_status_handler._log_lull_sampler_info(sampler_info, bundle_id)
with mock.patch('time.time') as time_mock:
time_mock.return_value = now + 42 * 60 # 21 minutes after previous one
- sampler_info = get_state_sampler_info_for_lull(21 * 60)
- self.fn_status_handler._log_lull_sampler_info(sampler_info)
+ bundle_id, sampler_info = get_state_sampler_info_for_lull(21 * 60)
+ self.fn_status_handler._log_lull_sampler_info(sampler_info, bundle_id)
class HeapDumpTest(unittest.TestCase):