You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/12/21 23:16:58 UTC
[1/2] incubator-beam git commit: Add support for date partitioned
table names
Repository: incubator-beam
Updated Branches:
refs/heads/python-sdk bb09c07b6 -> 409d067b3
Add support for date partitioned table names
These names have the format "tablename$YYYYmmdd". Previously the dollar
sign caused this to be deemed invalid.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a1af871a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a1af871a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a1af871a
Branch: refs/heads/python-sdk
Commit: a1af871a0c8c92a6d84f2e9950615f7737118d7e
Parents: bb09c07
Author: Kevin Graney <na...@gmail.com>
Authored: Tue Dec 6 15:09:42 2016 -0500
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Wed Dec 21 15:16:45 2016 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/io/bigquery.py | 6 ++++--
sdks/python/apache_beam/io/bigquery_test.py | 8 ++++++++
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a1af871a/sdks/python/apache_beam/io/bigquery.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/bigquery.py b/sdks/python/apache_beam/io/bigquery.py
index ce75e10..2059de4 100644
--- a/sdks/python/apache_beam/io/bigquery.py
+++ b/sdks/python/apache_beam/io/bigquery.py
@@ -275,7 +275,9 @@ def _parse_table_reference(table, dataset=None, project=None):
then the table argument must contain the entire table reference:
'DATASET.TABLE' or 'PROJECT:DATASET.TABLE'. This argument can be a
bigquery.TableReference instance in which case dataset and project are
- ignored and the reference is returned as a result.
+ ignored and the reference is returned as a result. Additionally, for date
+ partitioned tables, appending '$YYYYmmdd' to the table name is supported,
+ e.g. 'DATASET.TABLE$YYYYmmdd'.
dataset: The ID of the dataset containing this table or null if the table
reference is specified entirely by the table argument.
project: The ID of the project containing this table or null if the table
@@ -300,7 +302,7 @@ def _parse_table_reference(table, dataset=None, project=None):
# table name.
if dataset is None:
match = re.match(
- r'^((?P<project>.+):)?(?P<dataset>\w+)\.(?P<table>\w+)$', table)
+ r'^((?P<project>.+):)?(?P<dataset>\w+)\.(?P<table>[\w\$]+)$', table)
if not match:
raise ValueError(
'Expected a table reference (PROJECT:DATASET.TABLE or '
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a1af871a/sdks/python/apache_beam/io/bigquery_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/bigquery_test.py b/sdks/python/apache_beam/io/bigquery_test.py
index a2cf947..f6f9363 100644
--- a/sdks/python/apache_beam/io/bigquery_test.py
+++ b/sdks/python/apache_beam/io/bigquery_test.py
@@ -208,6 +208,14 @@ class TestBigQuerySource(unittest.TestCase):
self.assertEqual(source.query, 'my_query')
self.assertIsNone(source.table_reference)
+ def test_date_partitioned_table_name(self):
+ source = beam.io.BigQuerySource('dataset.table$20030102', validate=True)
+ dd = DisplayData.create_from(source)
+ expected_items = [
+ DisplayDataItemMatcher('validation', True),
+ DisplayDataItemMatcher('table', 'dataset.table$20030102')]
+ hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
+
class TestBigQuerySink(unittest.TestCase):
[2/2] incubator-beam git commit: Closes #1534
Posted by ro...@apache.org.
Closes #1534
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/409d067b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/409d067b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/409d067b
Branch: refs/heads/python-sdk
Commit: 409d067b36036981e330a055b652bb74a93f4ca2
Parents: bb09c07 a1af871
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Wed Dec 21 15:16:46 2016 -0800
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Wed Dec 21 15:16:46 2016 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/io/bigquery.py | 6 ++++--
sdks/python/apache_beam/io/bigquery_test.py | 8 ++++++++
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------