You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ri...@apache.org on 2023/06/01 19:13:35 UTC

[beam] branch master updated: Make lulls message in Python SDK to be consistent with the implementation in Java SDK (#26936)

This is an automated email from the ASF dual-hosted git repository.

riteshghorse pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 3dd239e2045 Make lulls message in Python SDK to be consistent with the implementation in Java SDK  (#26936)
3dd239e2045 is described below

commit 3dd239e204593dd8fc67ceb4aaa558768fc3edbb
Author: bzablocki <bz...@google.com>
AuthorDate: Thu Jun 1 21:13:25 2023 +0200

    Make lulls message in Python SDK to be consistent with the implementation in Java SDK  (#26936)
    
    * Make lulls message in Python SDK to be consistent with the implementation in Java SDK
    
    * Adjust unit tests to match introduced changes
    
    * Fix linting error
---
 .../apache_beam/runners/worker/worker_status.py    | 46 +++++++++++++---------
 .../runners/worker/worker_status_test.py           | 26 ++++++------
 2 files changed, 42 insertions(+), 30 deletions(-)

diff --git a/sdks/python/apache_beam/runners/worker/worker_status.py b/sdks/python/apache_beam/runners/worker/worker_status.py
index 40f0d927c0e..a7f4890344a 100644
--- a/sdks/python/apache_beam/runners/worker/worker_status.py
+++ b/sdks/python/apache_beam/runners/worker/worker_status.py
@@ -97,7 +97,7 @@ def heap_dump():
 
 
 def _state_cache_stats(state_cache):
-  #type: (StateCache) -> str
+  # type: (StateCache) -> str
 
   """Gather state cache statistics."""
   cache_stats = ['=' * 10 + ' CACHE STATS ' + '=' * 10]
@@ -236,34 +236,44 @@ class FnApiWorkerStatusHandler(object):
           processor = bundle_process_cache.lookup(instruction)
           if processor:
             info = processor.state_sampler.get_info()
-            self._log_lull_sampler_info(info)
+            self._log_lull_sampler_info(info, instruction)
 
-  def _log_lull_sampler_info(self, sampler_info):
+  def _log_lull_sampler_info(self, sampler_info, instruction):
     if not self._passed_lull_timeout_since_last_log():
       return
     if (sampler_info and sampler_info.time_since_transition and
         sampler_info.time_since_transition > self.log_lull_timeout_ns):
+      lull_seconds = sampler_info.time_since_transition / 1e9
+
       step_name = sampler_info.state_name.step_name
       state_name = sampler_info.state_name.name
-      lull_seconds = sampler_info.time_since_transition / 1e9
-      state_lull_log = (
-          'Operation ongoing for over %.2f seconds in state %s' %
-          (lull_seconds, state_name))
-      step_name_log = (' in step %s ' % step_name) if step_name else ''
-
-      exec_thread = getattr(sampler_info, 'tracked_thread', None)
-      if exec_thread is not None:
-        thread_frame = sys._current_frames().get(exec_thread.ident)  # pylint: disable=protected-access
-        stack_trace = '\n'.join(
-            traceback.format_stack(thread_frame)) if thread_frame else ''
+      if step_name and state_name:
+        step_name_log = (
+            ' for PTransform{name=%s, state=%s}' % (step_name, state_name))
       else:
-        stack_trace = '-NOT AVAILABLE-'
+        step_name_log = ''
+
+      stack_trace = self._get_stack_trace(sampler_info)
 
       _LOGGER.warning(
-          '%s%s without returning. Current Traceback:\n%s',
-          state_lull_log,
+          (
+              'Operation ongoing in bundle %s%s for at least %.2f seconds'
+              ' without outputting or completing.\n'
+              'Current Traceback:\n%s'),
+          instruction,
           step_name_log,
-          stack_trace)
+          lull_seconds,
+          stack_trace,
+      )
+
+  def _get_stack_trace(self, sampler_info):
+    exec_thread = getattr(sampler_info, 'tracked_thread', None)
+    if exec_thread is not None:
+      thread_frame = sys._current_frames().get(exec_thread.ident)  # pylint: disable=protected-access
+      return '\n'.join(
+          traceback.format_stack(thread_frame)) if thread_frame else ''
+    else:
+      return '-NOT AVAILABLE-'
 
   def _passed_lull_timeout_since_last_log(self) -> bool:
     if (time.time() - self._last_lull_logged_secs >
diff --git a/sdks/python/apache_beam/runners/worker/worker_status_test.py b/sdks/python/apache_beam/runners/worker/worker_status_test.py
index 7da533991df..1004d21e7fd 100644
--- a/sdks/python/apache_beam/runners/worker/worker_status_test.py
+++ b/sdks/python/apache_beam/runners/worker/worker_status_test.py
@@ -88,7 +88,7 @@ class FnApiWorkerStatusHandlerTest(unittest.TestCase):
 
   def test_log_lull_in_bundle_processor(self):
     def get_state_sampler_info_for_lull(lull_duration_s):
-      return statesampler.StateSamplerInfo(
+      return "bundle-id", statesampler.StateSamplerInfo(
           CounterName('progress-msecs', 'stage_name', 'step_name'),
           1,
           lull_duration_s * 1e9,
@@ -98,31 +98,33 @@ class FnApiWorkerStatusHandlerTest(unittest.TestCase):
     with mock.patch('logging.Logger.warning') as warn_mock:
       with mock.patch('time.time') as time_mock:
         time_mock.return_value = now
-        sampler_info = get_state_sampler_info_for_lull(21 * 60)
-        self.fn_status_handler._log_lull_sampler_info(sampler_info)
+        bundle_id, sampler_info = get_state_sampler_info_for_lull(21 * 60)
+        self.fn_status_handler._log_lull_sampler_info(sampler_info, bundle_id)
 
-        processing_template = warn_mock.call_args[0][1]
+        bundle_id_template = warn_mock.call_args[0][1]
         step_name_template = warn_mock.call_args[0][2]
-        traceback = warn_mock.call_args = warn_mock.call_args[0][3]
+        processing_template = warn_mock.call_args[0][3]
+        traceback = warn_mock.call_args = warn_mock.call_args[0][4]
 
-        self.assertIn('progress-msecs', processing_template)
+        self.assertIn('bundle-id', bundle_id_template)
         self.assertIn('step_name', step_name_template)
+        self.assertEqual(21 * 60, processing_template)
         self.assertIn('test_log_lull_in_bundle_processor', traceback)
 
       with mock.patch('time.time') as time_mock:
         time_mock.return_value = now + 6 * 60  # 6 minutes
-        sampler_info = get_state_sampler_info_for_lull(21 * 60)
-        self.fn_status_handler._log_lull_sampler_info(sampler_info)
+        bundle_id, sampler_info = get_state_sampler_info_for_lull(21 * 60)
+        self.fn_status_handler._log_lull_sampler_info(sampler_info, bundle_id)
 
       with mock.patch('time.time') as time_mock:
         time_mock.return_value = now + 21 * 60  # 21 minutes
-        sampler_info = get_state_sampler_info_for_lull(10 * 60)
-        self.fn_status_handler._log_lull_sampler_info(sampler_info)
+        bundle_id, sampler_info = get_state_sampler_info_for_lull(10 * 60)
+        self.fn_status_handler._log_lull_sampler_info(sampler_info, bundle_id)
 
       with mock.patch('time.time') as time_mock:
         time_mock.return_value = now + 42 * 60  # 21 minutes after previous one
-        sampler_info = get_state_sampler_info_for_lull(21 * 60)
-        self.fn_status_handler._log_lull_sampler_info(sampler_info)
+        bundle_id, sampler_info = get_state_sampler_info_for_lull(21 * 60)
+        self.fn_status_handler._log_lull_sampler_info(sampler_info, bundle_id)
 
 
 class HeapDumpTest(unittest.TestCase):