You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by an...@apache.org on 2023/06/07 20:12:50 UTC
[beam] branch master updated: Always make runner v2 default and return legacy container tag when runner_v1 is disabled (#27052)
This is an automated email from the ASF dual-hosted git repository.
anandinguva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3d7ced6e3d7 Always make runner v2 default and return legacy container tag when runner_v1 is disabled (#27052)
3d7ced6e3d7 is described below
commit 3d7ced6e3d7a849b7789bccb2f12b2b0e7acceaa
Author: Anand Inguva <34...@users.noreply.github.com>
AuthorDate: Wed Jun 7 16:12:41 2023 -0400
Always make runner v2 default and return legacy container tag when runner_v1 is disabled (#27052)
* Change the check for for runner_v2
* Add tests explicitly enabling runner v2
* change disable_runner_v2 to disable_runner_v2_until_v2.50
---
.../runners/dataflow/dataflow_runner.py | 1 +
.../runners/dataflow/internal/apiclient.py | 4 +-
.../runners/dataflow/internal/apiclient_test.py | 59 ++++++++++++++++++++--
3 files changed, 59 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 63bfc07cdc5..674d05c64ec 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -1413,6 +1413,7 @@ def _is_runner_v2_disabled(options):
return (
debug_options.lookup_experiment('disable_runner_v2') or
debug_options.lookup_experiment('disable_runner_v2_until_2023') or
+ debug_options.lookup_experiment('disable_runner_v2_until_v2.50') or
debug_options.lookup_experiment('disable_prime_runner_v2'))
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index f9140aed356..3f0bfcb395b 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -1202,12 +1202,12 @@ def get_container_image_from_options(pipeline_options):
Returns:
str: Container image for remote execution.
"""
- from apache_beam.runners.dataflow.dataflow_runner import _is_runner_v2
+ from apache_beam.runners.dataflow.dataflow_runner import _is_runner_v2_disabled
worker_options = pipeline_options.view_as(WorkerOptions)
if worker_options.sdk_container_image:
return worker_options.sdk_container_image
- is_runner_v2 = _is_runner_v2(pipeline_options)
+ is_runner_v2 = not _is_runner_v2_disabled(pipeline_options)
# Legacy and runner v2 exist in different repositories.
# Set to legacy format, override if runner v2
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index 21863d5f880..89ac58a727b 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -637,9 +637,28 @@ class UtilTest(unittest.TestCase):
sys.version_info[1],
names.BEAM_FNAPI_CONTAINER_VERSION)))
- # batch, legacy pipeline.
pipeline_options = PipelineOptions(
['--temp_location', 'gs://any-location/temp'])
+ env = apiclient.Environment(
+ [], #packages
+ pipeline_options,
+ '2.0.0', #any environment version
+ FAKE_PIPELINE_URL)
+ self.assertEqual(
+ env.proto.workerPools[0].workerHarnessContainerImage,
+ (
+ names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY +
+ '/beam_python%d.%d_sdk:%s' % (
+ sys.version_info[0],
+ sys.version_info[1],
+ names.BEAM_FNAPI_CONTAINER_VERSION)))
+
+ # batch, legacy pipeline.
+ pipeline_options = pipeline_options = PipelineOptions([
+ '--temp_location',
+ 'gs://any-location/temp',
+ '--experiments=disable_runner_v2_until_v2.50'
+ ])
env = apiclient.Environment(
[], #packages
pipeline_options,
@@ -673,9 +692,26 @@ class UtilTest(unittest.TestCase):
'/beam_python%d.%d_sdk:2.2.0' %
(sys.version_info[0], sys.version_info[1])))
- # batch, legacy pipeline.
pipeline_options = PipelineOptions(
['--temp_location', 'gs://any-location/temp'])
+ env = apiclient.Environment(
+ [], #packages
+ pipeline_options,
+ '2.0.0', #any environment version
+ FAKE_PIPELINE_URL)
+ self.assertEqual(
+ env.proto.workerPools[0].workerHarnessContainerImage,
+ (
+ names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY +
+ '/beam_python%d.%d_sdk:2.2.0' %
+ (sys.version_info[0], sys.version_info[1])))
+
+ # batch, legacy pipeline.
+ pipeline_options = pipeline_options = PipelineOptions([
+ '--temp_location',
+ 'gs://any-location/temp',
+ '--experiments=disable_runner_v2_until_v2.50'
+ ])
env = apiclient.Environment(
[], #packages
pipeline_options,
@@ -707,9 +743,26 @@ class UtilTest(unittest.TestCase):
'/beam_python%d.%d_sdk:2.2.0' %
(sys.version_info[0], sys.version_info[1])))
- # batch, legacy pipeline.
pipeline_options = PipelineOptions(
['--temp_location', 'gs://any-location/temp'])
+ env = apiclient.Environment(
+ [], #packages
+ pipeline_options,
+ '2.0.0', #any environment version
+ FAKE_PIPELINE_URL)
+ self.assertEqual(
+ env.proto.workerPools[0].workerHarnessContainerImage,
+ (
+ names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY +
+ '/beam_python%d.%d_sdk:2.2.0' %
+ (sys.version_info[0], sys.version_info[1])))
+
+ # batch, legacy pipeline
+ pipeline_options = pipeline_options = PipelineOptions([
+ '--temp_location',
+ 'gs://any-location/temp',
+ '--experiments=disable_runner_v2_until_v2.50'
+ ])
env = apiclient.Environment(
[], #packages
pipeline_options,