You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2024/03/08 21:47:17 UTC
(beam) branch master updated: [yaml] Add Beam YAML Examples (#30003)
This is an automated email from the ASF dual-hosted git repository.
robertwb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 53e8efa32d7 [yaml] Add Beam YAML Examples (#30003)
53e8efa32d7 is described below
commit 53e8efa32d7314b8dc764661d00479cd30a25b62
Author: Jeff Kinard <je...@thekinards.com>
AuthorDate: Fri Mar 8 16:47:09 2024 -0500
[yaml] Add Beam YAML Examples (#30003)
Signed-off-by: Jeffrey Kinard <je...@thekinards.com>
---
sdks/python/apache_beam/testing/test_pipeline.py | 7 +-
sdks/python/apache_beam/yaml/examples/README.md | 67 +++++++++
sdks/python/apache_beam/yaml/examples/__init__.py | 16 +++
.../apache_beam/yaml/examples/examples_test.py | 154 +++++++++++++++++++++
.../apache_beam/yaml/examples/regex_matches.yaml | 62 +++++++++
.../aggregation/combine_count_minimal.yaml | 62 +++++++++
.../aggregation/combine_max_minimal.yaml | 53 +++++++
.../aggregation/combine_mean_minimal.yaml | 53 +++++++
.../aggregation/combine_min_minimal.yaml | 53 +++++++
.../aggregation/combine_multiple_aggregations.yaml | 83 +++++++++++
.../transforms/aggregation/combine_sum.yaml | 67 +++++++++
.../aggregation/combine_sum_minimal.yaml | 53 +++++++
.../transforms/aggregation/group_into_batches.yaml | 67 +++++++++
.../aggregation/top_largest_per_key.yaml | 57 ++++++++
.../aggregation/top_smallest_per_key.yaml | 59 ++++++++
.../examples/transforms/elementwise/explode.yaml | 41 ++++++
.../transforms/elementwise/filter_callable.yaml | 52 +++++++
.../elementwise/map_to_fields_callable.yaml | 45 ++++++
.../yaml/examples/wordcount_minimal.yaml | 87 ++++++++++++
sdks/python/apache_beam/yaml/readme_test.py | 4 +-
20 files changed, 1138 insertions(+), 4 deletions(-)
diff --git a/sdks/python/apache_beam/testing/test_pipeline.py b/sdks/python/apache_beam/testing/test_pipeline.py
index 9f70e4d5c12..c8e6e7c3968 100644
--- a/sdks/python/apache_beam/testing/test_pipeline.py
+++ b/sdks/python/apache_beam/testing/test_pipeline.py
@@ -70,7 +70,8 @@ class TestPipeline(Pipeline):
argv=None,
is_integration_test=False,
blocking=True,
- additional_pipeline_args=None):
+ additional_pipeline_args=None,
+ display_data=None):
"""Initialize a pipeline object for test.
Args:
@@ -93,6 +94,8 @@ class TestPipeline(Pipeline):
completed.
additional_pipeline_args (List[str]): additional pipeline arguments to be
included when construction the pipeline options object.
+ display_data (Dict[str, Any]): a dictionary of static data associated
+ with this pipeline that can be displayed when it runs.
Raises:
ValueError: if either the runner or options argument is not
@@ -106,7 +109,7 @@ class TestPipeline(Pipeline):
self.blocking = blocking
if options is None:
options = PipelineOptions(self.options_list)
- super().__init__(runner, options)
+ super().__init__(runner, options, display_data=display_data)
def run(self, test_runner_api=True):
result = super().run(
diff --git a/sdks/python/apache_beam/yaml/examples/README.md b/sdks/python/apache_beam/yaml/examples/README.md
new file mode 100644
index 00000000000..0d76f41bef6
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/README.md
@@ -0,0 +1,67 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Examples Catalog
+
+<!-- TOC -->
+* [Examples Catalog](#examples-catalog)
+ * [Wordcount](#wordcount)
+ * [Transforms](#transforms)
+ * [Element-wise](#element-wise)
+ * [Aggregation](#aggregation)
+<!-- TOC -->
+
+This module contains a series of Beam YAML code samples that can be run using
+the command:
+```
+python -m apache_beam.yaml.main --pipeline_spec_file=/path/to/example.yaml
+```
+
+## Wordcount
+A good starting place is the [Wordcount](wordcount_minimal.yaml) example under
+the root example directory.
+This example reads in a text file, splits the text on each word, groups by each
+word, and counts the occurrence of each word. This is a classic example used in
+the other SDK's and shows off many of the functionalities of Beam YAML.
+
+## Transforms
+
+Examples in this directory show off the various built-in transforms of the Beam
+YAML framework.
+
+### Element-wise
+These examples leverage the built-in mapping transforms including `MapToFields`,
+`Filter` and `Explode`. More information can be found about mapping transforms
+[here](https://beam.apache.org/documentation/sdks/yaml-udf/).
+
+### Aggregation
+These examples leverage the built-in `Combine` transform for performing simple
+aggregations including sum, mean, count, etc.
+
+These examples are experimental and require that
+`yaml_experimental_features: Combine` be specified under the `options` tag, or
+by passing `--yaml_experimental_features=Combine` to the command to run the
+pipeline. i.e.
+```
+python -m apache_beam.yaml.main \
+ --pipeline_spec_file=/path/to/example.yaml \
+ --yaml_experimental_features=Combine
+```
+More information can be found about aggregation transforms
+[here](https://beam.apache.org/documentation/sdks/yaml-combine/).
\ No newline at end of file
diff --git a/sdks/python/apache_beam/yaml/examples/__init__.py b/sdks/python/apache_beam/yaml/examples/__init__.py
new file mode 100644
index 00000000000..cce3acad34a
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/sdks/python/apache_beam/yaml/examples/examples_test.py b/sdks/python/apache_beam/yaml/examples/examples_test.py
new file mode 100644
index 00000000000..e084f710a7c
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/examples_test.py
@@ -0,0 +1,154 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pytype: skip-file
+import glob
+import logging
+import os
+import random
+import unittest
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+from unittest import mock
+
+import yaml
+
+import apache_beam as beam
+from apache_beam import PCollection
+from apache_beam.examples.snippets.util import assert_matches_stdout
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.yaml import yaml_transform
+from apache_beam.yaml.readme_test import TestEnvironment
+from apache_beam.yaml.readme_test import replace_recursive
+
+
+def check_output(expected: List[str]):
+ def _check_inner(actual: PCollection[str]):
+ formatted_actual = actual | beam.Map(
+ lambda row: str(beam.Row(**row._asdict())))
+ assert_matches_stdout(formatted_actual, expected)
+
+ return _check_inner
+
+
+def create_test_method(
+ pipeline_spec_file: str,
+ custom_preprocessor: Optional[Callable[..., Union[Dict, List]]] = None):
+ @mock.patch('apache_beam.Pipeline', TestPipeline)
+ def test_yaml_example(self):
+ with open(pipeline_spec_file, encoding="utf-8") as f:
+ lines = f.readlines()
+ expected_key = '# Expected:\n'
+ if expected_key in lines:
+ expected = lines[lines.index('# Expected:\n') + 1:]
+ else:
+ raise ValueError(
+ f"Missing '# Expected:' tag in example file '{pipeline_spec_file}'")
+ for i, line in enumerate(expected):
+ expected[i] = line.replace('# ', '').replace('\n', '')
+ pipeline_spec = yaml.load(
+ ''.join(lines), Loader=yaml_transform.SafeLineLoader)
+
+ with TestEnvironment() as env:
+ if custom_preprocessor:
+ pipeline_spec = custom_preprocessor(pipeline_spec, expected, env)
+ with beam.Pipeline(options=PipelineOptions(
+ pickle_library='cloudpickle',
+ **yaml_transform.SafeLineLoader.strip_metadata(pipeline_spec.get(
+ 'options', {})))) as p:
+ actual = yaml_transform.expand_pipeline(p, pipeline_spec)
+ check_output(expected)(actual)
+
+ return test_yaml_example
+
+
+class YamlExamplesTestSuite:
+ _test_preprocessor: Dict[str, Callable[..., Union[Dict, List]]] = {}
+
+ def __init__(self, name: str, path: str):
+ self._test_suite = self.create_test_suite(name, path)
+
+ def run(self):
+ return self._test_suite
+
+ @classmethod
+ def parse_test_methods(cls, path: str):
+ files = glob.glob(path)
+ if not files and os.path.exists(path) and os.path.isfile(path):
+ files = [path]
+ for file in files:
+ test_name = f'test_{file.split(os.sep)[-1].replace(".", "_")}'
+ custom_preprocessor = cls._test_preprocessor.get(test_name, None)
+ yield test_name, create_test_method(file, custom_preprocessor)
+
+ @classmethod
+ def create_test_suite(cls, name: str, path: str):
+ return type(name, (unittest.TestCase, ), dict(cls.parse_test_methods(path)))
+
+ @classmethod
+ def register_test_preprocessor(cls, test_name: str):
+ def apply(preprocessor):
+ cls._test_preprocessor[test_name] = preprocessor
+ return preprocessor
+
+ return apply
+
+
+@YamlExamplesTestSuite.register_test_preprocessor('test_wordcount_minimal_yaml')
+def _wordcount_test_preprocessor(
+ test_spec: str, expected: List[str], env: TestEnvironment):
+ all_words = []
+ for element in expected:
+ word = element.split('=')[1].split(',')[0].replace("'", '')
+ count = int(element.split('=')[2].replace(')', ''))
+ all_words += [word] * count
+ random.shuffle(all_words)
+
+ lines = []
+ while all_words:
+ line_length = random.randint(1, min(10, len(all_words)))
+ line = " ".join(
+ all_words.pop(random.randrange(len(all_words)))
+ for _ in range(line_length))
+ lines.append(line)
+
+ return replace_recursive(
+ test_spec,
+ 'ReadFromText',
+ 'path',
+ env.input_file('kinglear.txt', '\n'.join(lines)))
+
+
+YAML_DOCS_DIR = os.path.join(os.path.dirname(__file__))
+ExamplesTest = YamlExamplesTestSuite(
+ 'ExamplesTest', os.path.join(YAML_DOCS_DIR, '*.yaml')).run()
+
+ElementWiseTest = YamlExamplesTestSuite(
+ 'ElementwiseExamplesTest',
+ os.path.join(YAML_DOCS_DIR, 'transforms/elementwise/*.yaml')).run()
+
+AggregationTest = YamlExamplesTestSuite(
+ 'AggregationExamplesTest',
+ os.path.join(YAML_DOCS_DIR, 'transforms/aggregation/*.yaml')).run()
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.INFO)
+ unittest.main()
diff --git a/sdks/python/apache_beam/yaml/examples/regex_matches.yaml b/sdks/python/apache_beam/yaml/examples/regex_matches.yaml
new file mode 100644
index 00000000000..176813cef6b
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/regex_matches.yaml
@@ -0,0 +1,62 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# pytype: skip-file
+
+# This pipline creates a series of {plant: description} key pairs, matches all
+# elements to a valid regex, filters out non-matching entries, then logs the
+# output.
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Garden plants
+ config:
+ elements:
+ - plant: 'π, Strawberry, perennial'
+ - plant: 'π₯, Carrot, biennial ignoring trailing words'
+ - plant: 'π, Eggplant, perennial'
+ - plant: 'π
, Tomato, annual'
+ - plant: 'π₯, Potato, perennial'
+ - plant: '# π, invalid, format'
+ - plant: 'invalid, π, format'
+ - type: MapToFields
+ name: Parse plants
+ config:
+ language: python
+ fields:
+ plant:
+ callable: |
+ import re
+ def regex_filter(row):
+ match = re.match("(?P<icon>[^\s,]+), *(\w+), *(\w+)", row.plant)
+ return match.group(0) if match else match
+
+ # Filters out None values produced by values that don't match regex
+ - type: Filter
+ config:
+ language: python
+ keep: plant
+ - type: LogForTesting
+
+# Expected:
+# Row(plant='π, Strawberry, perennial')
+# Row(plant='π₯, Carrot, biennial')
+# Row(plant='π, Eggplant, perennial')
+# Row(plant='π
, Tomato, annual')
+# Row(plant='π₯, Potato, perennial')
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_count_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_count_minimal.yaml
new file mode 100644
index 00000000000..b746e55e344
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_count_minimal.yaml
@@ -0,0 +1,62 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce names
+ config:
+ elements:
+ - season: 'spring'
+ produce: 'π'
+ - season: 'spring'
+ produce: 'π₯'
+ - season: 'summer'
+ produce: 'π₯'
+ - season: 'fall'
+ produce: 'π₯'
+ - season: 'spring'
+ produce: 'π'
+ - season: 'winter'
+ produce: 'π'
+ - season: 'spring'
+ produce: 'π
'
+ - season: 'summer'
+ produce: 'π
'
+ - season: 'fall'
+ produce: 'π
'
+ - season: 'summer'
+ produce: 'π½'
+ - type: Combine
+ name: Shortest names per key
+ config:
+ language: python
+ group_by: season
+ combine:
+ produce: count
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(season='spring', produce=4)
+# Row(season='summer', produce=3)
+# Row(season='fall', produce=2)
+# Row(season='winter', produce=1)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_max_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_max_minimal.yaml
new file mode 100644
index 00000000000..2588897a717
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_max_minimal.yaml
@@ -0,0 +1,53 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce
+ config:
+ elements:
+ - produce: 'π₯'
+ amount: 3
+ - produce: 'π₯'
+ amount: 2
+ - produce: 'π'
+ amount: 1
+ - produce: 'π
'
+ amount: 4
+ - produce: 'π
'
+ amount: 5
+ - produce: 'π
'
+ amount: 3
+ - type: Combine
+ name: Get max value per key
+ config:
+ language: python
+ group_by: produce
+ combine:
+ amount: max
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(produce='π₯', amount=3)
+# Row(produce='π', amount=1)
+# Row(produce='π
', amount=5)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_mean_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_mean_minimal.yaml
new file mode 100644
index 00000000000..b00b7e817ab
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_mean_minimal.yaml
@@ -0,0 +1,53 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce
+ config:
+ elements:
+ - produce: 'π₯'
+ amount: 3
+ - produce: 'π₯'
+ amount: 2
+ - produce: 'π'
+ amount: 1
+ - produce: 'π
'
+ amount: 4
+ - produce: 'π
'
+ amount: 5
+ - produce: 'π
'
+ amount: 3
+ - type: Combine
+ name: Get mean value per key
+ config:
+ language: python
+ group_by: produce
+ combine:
+ amount: mean
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(produce='π₯', amount=2.5)
+# Row(produce='π', amount=1.0)
+# Row(produce='π
', amount=4.0)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_min_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_min_minimal.yaml
new file mode 100644
index 00000000000..1e05fd3f755
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_min_minimal.yaml
@@ -0,0 +1,53 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce
+ config:
+ elements:
+ - produce: 'π₯'
+ amount: 3
+ - produce: 'π₯'
+ amount: 2
+ - produce: 'π'
+ amount: 1
+ - produce: 'π
'
+ amount: 4
+ - produce: 'π
'
+ amount: 5
+ - produce: 'π
'
+ amount: 3
+ - type: Combine
+ name: Get min value per key
+ config:
+ language: python
+ group_by: produce
+ combine:
+ amount: min
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(produce='π₯', amount=2)
+# Row(produce='π', amount=1)
+# Row(produce='π
', amount=3)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_multiple_aggregations.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_multiple_aggregations.yaml
new file mode 100644
index 00000000000..1263ac21fac
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_multiple_aggregations.yaml
@@ -0,0 +1,83 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce
+ config:
+ elements:
+ - recipe: 'pie'
+ fruit: 'raspberry'
+ quantity: 1
+ unit_price: 3.50
+ - recipe: 'pie'
+ fruit: 'blackberry'
+ quantity: 1
+ unit_price: 4.00
+ - recipe: 'pie'
+ fruit: 'blueberry'
+ quantity: 1
+ unit_price: 2.00
+ - recipe: 'muffin'
+ fruit: 'blueberry'
+ quantity: 2
+ unit_price: 2.00
+ - recipe: 'muffin'
+ fruit: 'banana'
+ quantity: 3
+ unit_price: 1.00
+ # Simulates global GroupBy by creating global key
+ - type: MapToFields
+ name: Add global key
+ config:
+ append: true
+ language: python
+ fields:
+ global_key: '0'
+ - type: Combine
+ name: Get multiple aggregations per key
+ config:
+ language: python
+ group_by: global_key
+ combine:
+ min_price:
+ value: unit_price
+ fn: min
+ mean_price:
+ value: unit_price
+ fn: mean
+ max_price:
+ value: unit_price
+ fn: max
+ # Removes unwanted global key
+ - type: MapToFields
+ name: Remove global key
+ config:
+ fields:
+ min_price: min_price
+ mean_price: mean_price
+ max_price: max_price
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(min_price=1.0, mean_price=2.5, max_price=4.0)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum.yaml
new file mode 100644
index 00000000000..77cd1529b58
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum.yaml
@@ -0,0 +1,67 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce
+ config:
+ elements:
+ - recipe: 'pie'
+ fruit: 'raspberry'
+ quantity: 1
+ unit_price: 3.50
+ - recipe: 'pie'
+ fruit: 'blackberry'
+ quantity: 1
+ unit_price: 4.00
+ - recipe: 'pie'
+ fruit: 'blueberry'
+ quantity: 1
+ unit_price: 2.00
+ - recipe: 'muffin'
+ fruit: 'blueberry'
+ quantity: 2
+ unit_price: 3.00
+ - recipe: 'muffin'
+ fruit: 'banana'
+ quantity: 3
+ unit_price: 1.00
+ - type: Combine
+ name: Sum values per key
+ config:
+ language: python
+ group_by: fruit
+ combine:
+ total_quantity:
+ value: quantity
+ fn: sum
+ mean_price:
+ value: unit_price
+ fn: mean
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(fruit='raspberry', total_quantity=1, mean_price=3.5)
+# Row(fruit='blackberry', total_quantity=1, mean_price=4.0)
+# Row(fruit='blueberry', total_quantity=3, mean_price=2.5)
+# Row(fruit='banana', total_quantity=3, mean_price=1.0)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum_minimal.yaml
new file mode 100644
index 00000000000..905a30d13f0
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum_minimal.yaml
@@ -0,0 +1,53 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce
+ config:
+ elements:
+ - produce: 'π₯'
+ amount: 3
+ - produce: 'π₯'
+ amount: 2
+ - produce: 'π'
+ amount: 1
+ - produce: 'π
'
+ amount: 4
+ - produce: 'π
'
+ amount: 5
+ - produce: 'π
'
+ amount: 3
+ - type: Combine
+ name: Count elements per key
+ config:
+ language: python
+ group_by: produce
+ combine:
+ amount: sum
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(produce='π₯', amount=5)
+# Row(produce='π', amount=1)
+# Row(produce='π
', amount=12)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/group_into_batches.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/group_into_batches.yaml
new file mode 100644
index 00000000000..a956ea604c6
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/group_into_batches.yaml
@@ -0,0 +1,67 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce
+ config:
+ elements:
+ - season: 'spring'
+ produce: 'π'
+ - season: 'spring'
+ produce: 'π₯'
+ - season: 'spring'
+ produce: 'π'
+ - season: 'spring'
+ produce: 'π
'
+ - season: 'summer'
+ produce: 'π₯'
+ - season: 'summer'
+ produce: 'π
'
+ - season: 'summer'
+ produce: 'π½'
+ - season: 'fall'
+ produce: 'π₯'
+ - season: 'fall'
+ produce: 'π
'
+ - season: 'winter'
+ produce: 'π'
+ - type: Combine
+ name: Group into batches
+ config:
+ language: python
+ group_by: season
+ combine:
+ produce:
+ value: produce
+ fn:
+ type: 'apache_beam.transforms.combiners.TopCombineFn'
+ config:
+ n: 3
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(season='spring', produce=['π₯', 'π', 'π'])
+# Row(season='summer', produce=['π₯', 'π
', 'π½'])
+# Row(season='fall', produce=['π₯', 'π
'])
+# Row(season='winter', produce=['π'])
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_largest_per_key.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_largest_per_key.yaml
new file mode 100644
index 00000000000..c35a5da1bd4
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_largest_per_key.yaml
@@ -0,0 +1,57 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce
+ config:
+ elements:
+ - produce: 'π₯'
+ amount: 3
+ - produce: 'π₯'
+ amount: 2
+ - produce: 'π'
+ amount: 1
+ - produce: 'π
'
+ amount: 4
+ - produce: 'π
'
+ amount: 5
+ - produce: 'π
'
+ amount: 3
+ - type: Combine
+ config:
+ language: python
+ group_by: produce
+ combine:
+ biggest:
+ value: amount
+ fn:
+ type: 'apache_beam.transforms.combiners.TopCombineFn'
+ config:
+ n: 2
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(produce='π₯', biggest=[3, 2])
+# Row(produce='π', biggest=[1])
+# Row(produce='π
', biggest=[5, 4])
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_smallest_per_key.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_smallest_per_key.yaml
new file mode 100644
index 00000000000..ebd7c0c34fe
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_smallest_per_key.yaml
@@ -0,0 +1,59 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Create produce
+ config:
+ elements:
+ - produce: 'π₯'
+ amount: 3
+ - produce: 'π₯'
+ amount: 2
+ - produce: 'π'
+ amount: 1
+ - produce: 'π
'
+ amount: 4
+ - produce: 'π
'
+ amount: 5
+ - produce: 'π
'
+ amount: 3
+ - type: Combine
+ name: Smallest N values per key
+ config:
+ language: python
+ group_by: produce
+ combine:
+ smallest:
+ value: amount
+ fn:
+ type: 'apache_beam.transforms.combiners.TopCombineFn'
+ config:
+ n: 2
+ reverse: true
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(produce='π₯', smallest=[2, 3])
+# Row(produce='π', smallest=[1])
+# Row(produce='π
', smallest=[3, 4])
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/elementwise/explode.yaml b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/explode.yaml
new file mode 100644
index 00000000000..70432ee7735
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/explode.yaml
@@ -0,0 +1,41 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the 'License'); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Gardening plants
+ config:
+ elements:
+ - produce: ['πStrawberry', 'π₯Potato']
+ season: 'spring'
+ - produce: ['π₯Carrot', 'πEggplant', 'π
Tomato']
+ season: 'summer'
+ - type: Explode
+ name: Flatten lists
+ config:
+ fields: [produce]
+ - type: LogForTesting
+
+# Expected:
+# Row(produce='πStrawberry', season='spring')
+# Row(produce='π₯Carrot', season='summer')
+# Row(produce='πEggplant', season='summer')
+# Row(produce='π
Tomato', season='summer')
+# Row(produce='π₯Potato', season='spring')
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/elementwise/filter_callable.yaml b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/filter_callable.yaml
new file mode 100644
index 00000000000..f66f672fd59
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/filter_callable.yaml
@@ -0,0 +1,52 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Gardening plants
+ config:
+ elements:
+ - 'icon': 'π'
+ 'name': 'Strawberry'
+ 'duration': 'perennial'
+ - 'icon': 'π₯'
+ 'name': 'Carrot'
+ 'duration': 'biennial'
+ - 'icon': 'π'
+ 'name': 'Eggplant'
+ 'duration': 'perennial'
+ - 'icon': 'π
'
+ 'name': 'Tomato'
+ 'duration': 'annual'
+ - 'icon': 'π₯'
+ 'name': 'Potato'
+ 'duration': 'perennial'
+ - type: Filter
+ name: Filter perennials
+ config:
+ language: python
+ keep:
+ callable: "lambda plant: plant.duration == 'perennial'"
+ - type: LogForTesting
+
+# Expected:
+# Row(icon='π', name='Strawberry', duration='perennial')
+# Row(icon='π', name='Eggplant', duration='perennial')
+# Row(icon='π₯', name='Potato', duration='perennial')
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/elementwise/map_to_fields_callable.yaml b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/map_to_fields_callable.yaml
new file mode 100644
index 00000000000..a6468b8829f
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/map_to_fields_callable.yaml
@@ -0,0 +1,45 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+ type: chain
+ transforms:
+ - type: Create
+ name: Gardening plants
+ config:
+ elements:
+ - '# πStrawberry'
+ - '# π₯Carrot'
+ - '# πEggplant'
+ - '# π
Tomato'
+ - '# π₯Potato'
+ - type: MapToFields
+ name: Strip header
+ config:
+ language: python
+ fields:
+ element:
+ callable: "lambda row: row.element.strip('# \\n')"
+ - type: LogForTesting
+
+# Expected:
+# Row(element='πStrawberry')
+# Row(element='π₯Carrot')
+# Row(element='πEggplant')
+# Row(element='π
Tomato')
+# Row(element='π₯Potato')
diff --git a/sdks/python/apache_beam/yaml/examples/wordcount_minimal.yaml b/sdks/python/apache_beam/yaml/examples/wordcount_minimal.yaml
new file mode 100644
index 00000000000..044db2790ac
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/wordcount_minimal.yaml
@@ -0,0 +1,87 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the# Row(word='License'); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an# Row(word='AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This examples reads from a public file stores on Google Cloud. This
+# requires authenticating with Google Cloud, or setting the file in
+#`ReadFromText` to a local file.
+#
+# To set up Application Default Credentials,
+# see https://cloud.google.com/docs/authentication/external/set-up-adc for more
+# information
+#
+# This pipeline reads in a text file, counts distinct words found in the text,
+# then logs a row containing each word and its count.
+pipeline:
+ type: chain
+ transforms:
+
+ # Read text file into a collection of rows, each with one field, "line"
+ - type: ReadFromText
+ config:
+ path: gs://dataflow-samples/shakespeare/kinglear.txt
+
+ # Split line field in each row into list of words
+ - type: MapToFields
+ config:
+ language: python
+ fields:
+ words:
+ callable: |
+ import re
+ def my_mapping(row):
+ return re.findall(r"[A-Za-z\']+", row.line.lower())
+
+ # Explode each list of words into separate rows
+ - type: Explode
+ config:
+ fields: words
+
+ # Since each word is now distinct row, rename field to "word"
+ - type: MapToFields
+ config:
+ fields:
+ word: words
+
+ # Group by distinct words in the collection and add field "count" that
+ # contains number of instances, or count, for each word in the collection.
+ - type: Combine
+ config:
+ language: python
+ group_by: word
+ combine:
+ count:
+ value: word
+ fn: count
+
+ # Log out results
+ - type: LogForTesting
+
+options:
+ yaml_experimental_features: Combine
+
+# Expected:
+# Row(word='king', count=311)
+# Row(word='lear', count=253)
+# Row(word='dramatis', count=1)
+# Row(word='personae', count=1)
+# Row(word='of', count=483)
+# Row(word='britain', count=2)
+# Row(word='france', count=32)
+# Row(word='duke', count=26)
+# Row(word='burgundy', count=20)
+# Row(word='cornwall', count=75)
diff --git a/sdks/python/apache_beam/yaml/readme_test.py b/sdks/python/apache_beam/yaml/readme_test.py
index 9a0e4454611..4ca60e6176b 100644
--- a/sdks/python/apache_beam/yaml/readme_test.py
+++ b/sdks/python/apache_beam/yaml/readme_test.py
@@ -191,10 +191,10 @@ def replace_recursive(spec, transform_type, arg_name, arg_value):
def create_test_method(test_type, test_name, test_yaml):
- test_yaml = test_yaml.replace(
- 'pkg.module.', 'apache_beam.yaml.readme_test._Fakes.')
test_yaml = test_yaml.replace(
'apache_beam.pkg.module.', 'apache_beam.yaml.readme_test._Fakes.')
+ test_yaml = test_yaml.replace(
+ 'pkg.module.', 'apache_beam.yaml.readme_test._Fakes.')
def test(self):
with TestEnvironment() as env: