You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2022/05/06 17:55:02 UTC
[beam] branch master updated: [BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 68e2a105a3f [BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors
new a0684f502f0 Merge pull request #17566 from [BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors
68e2a105a3f is described below
commit 68e2a105a3f6d552b58eaec2cdc1b7c01296eb0f
Author: Heejong Lee <he...@gmail.com>
AuthorDate: Thu May 5 13:30:45 2022 -0700
[BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors
---
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 18 +++---------------
sdks/python/apache_beam/io/gcp/bigquery_tools_test.py | 4 ++--
2 files changed, 5 insertions(+), 17 deletions(-)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
index 676755d90f9..89efa1ef623 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
@@ -502,10 +502,9 @@ class BigQueryWrapper(object):
job_labels=None):
if not source_uris and not source_stream:
- raise ValueError(
- 'Either a non-empty list of fully-qualified source URIs must be '
- 'provided via the source_uris parameter or an open file object must '
- 'be provided via the source_stream parameter. Got neither.')
+ _LOGGER.warning(
+ 'Both source URIs and source stream are not provided. BigQuery load '
+ 'job will not load any data.')
if source_uris and source_stream:
raise ValueError(
@@ -1002,17 +1001,6 @@ class BigQueryWrapper(object):
Returns:
bigquery.JobReference with the information about the job that was started.
"""
- if not source_uris and not source_stream:
- raise ValueError(
- 'Either a non-empty list of fully-qualified source URIs must be '
- 'provided via the source_uris parameter or an open file object must '
- 'be provided via the source_stream parameter. Got neither.')
-
- if source_uris and source_stream:
- raise ValueError(
- 'Only one of source_uris and source_stream may be specified. '
- 'Got both.')
-
project_id = (
destination.projectId
if load_job_project_id is None else load_job_project_id)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
index c2d56f56de7..db6abf8ded5 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py
@@ -443,8 +443,8 @@ class TestBigQueryWrapper(unittest.TestCase):
source_stream=io.BytesIO())
# Neither source_uri nor source_stream specified.
- with self.assertRaises(ValueError):
- wrapper.perform_load_job(destination='P:D.T', job_id='J')
+ wrapper.perform_load_job(
+ destination=parse_table_reference('project:dataset.table'), job_id='J')
def test_perform_load_job_with_source_stream(self):
client = mock.Mock()