You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yi...@apache.org on 2022/05/10 00:15:11 UTC
[beam] branch release-2.39.0 updated: Cherry-pick request #17566 from [BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors (#17595)
This is an automated email from the ASF dual-hosted git repository.
yichi pushed a commit to branch release-2.39.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.39.0 by this push:
new 6e2cca13c2b Cherry-pick request #17566 from [BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors (#17595)
6e2cca13c2b is described below
commit 6e2cca13c2b319c35db1e63104d8601a054bde65
Author: Yichi Zhang <zy...@google.com>
AuthorDate: Mon May 9 17:15:04 2022 -0700
Cherry-pick request #17566 from [BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors (#17595)
[BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors
Co-authored-by: Pablo <pa...@users.noreply.github.com>
---
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 18 +++---------------
sdks/python/apache_beam/io/gcp/bigquery_tools_test.py | 4 ++--
2 files changed, 5 insertions(+), 17 deletions(-)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
index 676755d90f9..89efa1ef623 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
@@ -502,10 +502,9 @@ class BigQueryWrapper(object):
job_labels=None):
if not source_uris and not source_stream:
- raise ValueError(
- 'Either a non-empty list of fully-qualified source URIs must be '
- 'provided via the source_uris parameter or an open file object must '
- 'be provided via the source_stream parameter. Got neither.')
+ _LOGGER.warning(
+ 'Both source URIs and source stream are not provided. BigQuery load '
+ 'job will not load any data.')
if source_uris and source_stream:
raise ValueError(
@@ -1002,17 +1001,6 @@ class BigQueryWrapper(object):
Returns:
bigquery.JobReference with the information about the job that was started.
"""
- if not source_uris and not source_stream:
- raise ValueError(
- 'Either a non-empty list of fully-qualified source URIs must be '
- 'provided via the source_uris parameter or an open file object must '
- 'be provided via the source_stream parameter. Got neither.')
-
- if source_uris and source_stream:
- raise ValueError(
- 'Only one of source_uris and source_stream may be specified. '
- 'Got both.')
-
project_id = (
destination.projectId
if load_job_project_id is None else load_job_project_id)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
index c2d56f56de7..db6abf8ded5 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
@@ -443,8 +443,8 @@ class TestBigQueryWrapper(unittest.TestCase):
source_stream=io.BytesIO())
# Neither source_uri nor source_stream specified.
- with self.assertRaises(ValueError):
- wrapper.perform_load_job(destination='P:D.T', job_id='J')
+ wrapper.perform_load_job(
+ destination=parse_table_reference('project:dataset.table'), job_id='J')
def test_perform_load_job_with_source_stream(self):
client = mock.Mock()