[beam] branch master updated: Option to skip display operations in beam's InteractiveRunner. This helps significantly when running large pipelines that do not need display (ie, that are not run on Jupyter/Colab).

[pa...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 1baad3b  Option to skip display operations in beam's InteractiveRunner. This helps significantly when running large pipelines that do not need display (ie, that are not run on Jupyter/Colab).
     new 0a75cb6  Merge pull request #10015 from andrefaraujo/master Option to skip display operations in beam's InteractiveRunner.
1baad3b is described below

commit 1baad3bb5a7c0c61d5f31c2bf64d1fad937e008e
Author: Andre Araujo <an...@google.com>
AuthorDate: Wed Nov 6 13:16:07 2019 -0800

    Option to skip display operations in beam's InteractiveRunner. This helps significantly when running large pipelines that do not need display (ie, that are not run on Jupyter/Colab).
---
 .../runners/interactive/interactive_runner.py      | 25 +++++++++++++++-------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/sdks/python/apache_beam/runners/interactive/interactive_runner.py b/sdks/python/apache_beam/runners/interactive/interactive_runner.py
index 94c0de7..7b6df70 100644
--- a/sdks/python/apache_beam/runners/interactive/interactive_runner.py
+++ b/sdks/python/apache_beam/runners/interactive/interactive_runner.py
@@ -48,7 +48,8 @@ class InteractiveRunner(runners.PipelineRunner):
                underlying_runner=None,
                cache_dir=None,
                cache_format='text',
-               render_option=None):
+               render_option=None,
+               skip_display=False):
     """Constructor of InteractiveRunner.
 
     Args:
@@ -58,12 +59,16 @@ class InteractiveRunner(runners.PipelineRunner):
           PCollection caches. Available options are 'text' and 'tfrecord'.
       render_option: (str) this parameter decides how the pipeline graph is
           rendered. See display.pipeline_graph_renderer for available options.
+      skip_display: (bool) whether to skip display operations when running the
+          pipeline. Useful if running large pipelines when display is not
+          needed.
     """
     self._underlying_runner = (underlying_runner
                                or direct_runner.DirectRunner())
     self._cache_manager = cache.FileBasedCacheManager(cache_dir, cache_format)
     self._renderer = pipeline_graph_renderer.get_renderer(render_option)
     self._in_session = False
+    self._skip_display = skip_display
 
   def is_fnapi_compatible(self):
     # TODO(BEAM-8436): return self._underlying_runner.is_fnapi_compatible()
@@ -140,15 +145,19 @@ class InteractiveRunner(runners.PipelineRunner):
         self._underlying_runner,
         options)
 
-    display = display_manager.DisplayManager(
-        pipeline_proto=pipeline_proto,
-        pipeline_analyzer=analyzer,
-        cache_manager=self._cache_manager,
-        pipeline_graph_renderer=self._renderer)
-    display.start_periodic_update()
+    if not self._skip_display:
+      display = display_manager.DisplayManager(
+          pipeline_proto=pipeline_proto,
+          pipeline_analyzer=analyzer,
+          cache_manager=self._cache_manager,
+          pipeline_graph_renderer=self._renderer)
+      display.start_periodic_update()
+
     result = pipeline_to_execute.run()
     result.wait_until_finish()
-    display.stop_periodic_update()
+
+    if not self._skip_display:
+      display.stop_periodic_update()
 
     return PipelineResult(result, self, self._analyzer.pipeline_info(),
                           self._cache_manager, pcolls_to_pcoll_id)