You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2017/02/11 01:40:43 UTC
[1/2] beam git commit: Updates BigQuery read transform so that
DirectRunner handles 'null' fields properly.
Repository: beam
Updated Branches:
refs/heads/master af02b8c32 -> a2826195b
Updates BigQuery read transform so that DirectRunner handles 'null' fields properly.
Before this change, for DirectRunner, a record (dictionary) returned by BigQuery read transform will not contain keys for fields that are 'null'. For DataflowRunner, these fields will be available with value 'None'.
I believe, retaining these fields value 'None' to be the proper behavior here.
This change makes these two runners consistent when it comes to handling BigQuery 'null' values.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/944ed4a3
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/944ed4a3
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/944ed4a3
Branch: refs/heads/master
Commit: 944ed4a3ecadf4ce3c6825fac1d13d775ccc74b9
Parents: af02b8c
Author: Chamikara Jayalath <ch...@google.com>
Authored: Fri Feb 10 14:19:53 2017 -0800
Committer: Ahmet Altay <al...@google.com>
Committed: Fri Feb 10 17:40:26 2017 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/io/bigquery.py | 13 +++++++------
sdks/python/apache_beam/io/bigquery_test.py | 11 ++++++++---
2 files changed, 15 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/944ed4a3/sdks/python/apache_beam/io/bigquery.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/bigquery.py b/sdks/python/apache_beam/io/bigquery.py
index 8f55f42..93033ea 100644
--- a/sdks/python/apache_beam/io/bigquery.py
+++ b/sdks/python/apache_beam/io/bigquery.py
@@ -1065,17 +1065,18 @@ class BigQueryWrapper(object):
value = None
if isinstance(schema, bigquery.TableSchema):
cell = row.f[index]
- if cell.v is None:
- continue # Field not present in the row.
- value = from_json_value(cell.v)
+ value = from_json_value(cell.v) if cell.v is not None else None
elif isinstance(schema, bigquery.TableFieldSchema):
cell = row['f'][index]
- if 'v' not in cell:
- continue # Field not present in the row.
- value = cell['v']
+ value = cell['v'] if 'v' in cell else None
if field.mode == 'REPEATED':
result[field.name] = [self._convert_cell_value_to_dict(x['v'], field)
for x in value]
+ elif value is None:
+ if not field.mode == 'NULLABLE':
+ raise ValueError('Received \'None\' as the value for the field %s '
+ 'but the field is not NULLABLE.', field.name)
+ result[field.name] = None
else:
result[field.name] = self._convert_cell_value_to_dict(value, field)
return result
http://git-wip-us.apache.org/repos/asf/beam/blob/944ed4a3/sdks/python/apache_beam/io/bigquery_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/bigquery_test.py b/sdks/python/apache_beam/io/bigquery_test.py
index cdca884..b64121a 100644
--- a/sdks/python/apache_beam/io/bigquery_test.py
+++ b/sdks/python/apache_beam/io/bigquery_test.py
@@ -308,14 +308,19 @@ class TestBigQueryReader(unittest.TestCase):
'ts': '22:39:12.627498',
'dt_ts': '2008-12-25T07:30:00',
'r': {'s2': 'b'},
- 'rpr': [{'s3': 'c', 'rpr2': [{'rs': ['d', 'e'], 's4': 'f'}]}]
+ 'rpr': [{'s3': 'c', 'rpr2': [{'rs': ['d', 'e'], 's4': None}]}]
},
{
'i': 10,
's': 'xyz',
'f': -3.14,
'b': False,
- 'rpr': []
+ 'rpr': [],
+ 't': None,
+ 'dt': None,
+ 'ts': None,
+ 'dt_ts': None,
+ 'r': None,
}]
nested_schema = [
@@ -372,7 +377,7 @@ class TestBigQueryReader(unittest.TestCase):
# schemas correctly so we have to use this f,v based format
bigquery.TableCell(v=to_json_value({'f': [{'v': 'b'}]})),
bigquery.TableCell(v=to_json_value([{'v':{'f':[{'v':'c'}, {'v':[
- {'v':{'f':[{'v':[{'v':'d'}, {'v':'e'}]}, {'v':'f'}]}}]}]}}]))
+ {'v':{'f':[{'v':[{'v':'d'}, {'v':'e'}]}, {'v':None}]}}]}]}}]))
]),
bigquery.TableRow(f=[
bigquery.TableCell(v=to_json_value('false')),
[2/2] beam git commit: This closes #1978
Posted by al...@apache.org.
This closes #1978
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a2826195
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a2826195
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a2826195
Branch: refs/heads/master
Commit: a2826195b62299cbf9ee85ae746c79d287d49ad2
Parents: af02b8c 944ed4a
Author: Ahmet Altay <al...@google.com>
Authored: Fri Feb 10 17:40:30 2017 -0800
Committer: Ahmet Altay <al...@google.com>
Committed: Fri Feb 10 17:40:30 2017 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/io/bigquery.py | 13 +++++++------
sdks/python/apache_beam/io/bigquery_test.py | 11 ++++++++---
2 files changed, 15 insertions(+), 9 deletions(-)
----------------------------------------------------------------------