You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2023/09/08 17:30:23 UTC
[beam] 01/01: Merge pull request #28169 [YAML] Implement a consistent BigQuery read and write transform.
This is an automated email from the ASF dual-hosted git repository.
robertwb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 778c30eb9a3931d1c41f427b400b435ab611bef4
Merge: ffb9d113ad9 63c16bbbe42
Author: Robert Bradshaw <ro...@gmail.com>
AuthorDate: Fri Sep 8 10:30:15 2023 -0700
Merge pull request #28169 [YAML] Implement a consistent BigQuery read and write transform.
...ueryStorageWriteApiSchemaTransformProvider.java | 133 ++++++++++++++-------
...StorageWriteApiSchemaTransformProviderTest.java | 28 ++++-
sdks/python/apache_beam/io/gcp/bigquery.py | 96 +++++++++------
sdks/python/apache_beam/pipeline.py | 5 +-
sdks/python/apache_beam/transforms/external.py | 108 +++++++++++------
.../python/apache_beam/transforms/external_test.py | 10 +-
.../apache_beam/typehints/schema_registry.py | 7 +-
sdks/python/apache_beam/yaml/main.py | 10 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 53 ++++++++
sdks/python/apache_beam/yaml/yaml_io.py | 116 ++++++++++++++++++
sdks/python/apache_beam/yaml/yaml_provider.py | 92 +++++++++++---
sdks/python/apache_beam/yaml/yaml_transform.py | 5 +-
12 files changed, 525 insertions(+), 138 deletions(-)
diff --cc sdks/python/apache_beam/yaml/yaml_provider.py
index 61f77c8d36b,f7a237bc045..d42d7aaffee
--- a/sdks/python/apache_beam/yaml/yaml_provider.py
+++ b/sdks/python/apache_beam/yaml/yaml_provider.py
@@@ -468,15 -452,10 +491,9 @@@ def create_builtin_provider()
# TODO: Triggering, etc.
return beam.WindowInto(window_fn)
- ios = {
- key: getattr(apache_beam.io, key)
- for key in dir(apache_beam.io)
- if key.startswith('ReadFrom') or key.startswith('WriteTo')
- }
-
return InlineProvider(
dict({
- 'Create': lambda elements,
- reshuffle=True: beam.Create(elements, reshuffle),
+ 'Create': create,
'PyMap': lambda fn: beam.Map(
python_callable.PythonCallableWithSource(fn)),
'PyMapTuple': lambda fn: beam.MapTuple(