You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2024/03/08 21:47:17 UTC

(beam) branch master updated: [yaml] Add Beam YAML Examples (#30003)

This is an automated email from the ASF dual-hosted git repository.

robertwb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 53e8efa32d7 [yaml] Add Beam YAML Examples (#30003)
53e8efa32d7 is described below

commit 53e8efa32d7314b8dc764661d00479cd30a25b62
Author: Jeff Kinard <je...@thekinards.com>
AuthorDate: Fri Mar 8 16:47:09 2024 -0500

    [yaml] Add Beam YAML Examples (#30003)
    
    Signed-off-by: Jeffrey Kinard <je...@thekinards.com>
---
 sdks/python/apache_beam/testing/test_pipeline.py   |   7 +-
 sdks/python/apache_beam/yaml/examples/README.md    |  67 +++++++++
 sdks/python/apache_beam/yaml/examples/__init__.py  |  16 +++
 .../apache_beam/yaml/examples/examples_test.py     | 154 +++++++++++++++++++++
 .../apache_beam/yaml/examples/regex_matches.yaml   |  62 +++++++++
 .../aggregation/combine_count_minimal.yaml         |  62 +++++++++
 .../aggregation/combine_max_minimal.yaml           |  53 +++++++
 .../aggregation/combine_mean_minimal.yaml          |  53 +++++++
 .../aggregation/combine_min_minimal.yaml           |  53 +++++++
 .../aggregation/combine_multiple_aggregations.yaml |  83 +++++++++++
 .../transforms/aggregation/combine_sum.yaml        |  67 +++++++++
 .../aggregation/combine_sum_minimal.yaml           |  53 +++++++
 .../transforms/aggregation/group_into_batches.yaml |  67 +++++++++
 .../aggregation/top_largest_per_key.yaml           |  57 ++++++++
 .../aggregation/top_smallest_per_key.yaml          |  59 ++++++++
 .../examples/transforms/elementwise/explode.yaml   |  41 ++++++
 .../transforms/elementwise/filter_callable.yaml    |  52 +++++++
 .../elementwise/map_to_fields_callable.yaml        |  45 ++++++
 .../yaml/examples/wordcount_minimal.yaml           |  87 ++++++++++++
 sdks/python/apache_beam/yaml/readme_test.py        |   4 +-
 20 files changed, 1138 insertions(+), 4 deletions(-)

diff --git a/sdks/python/apache_beam/testing/test_pipeline.py b/sdks/python/apache_beam/testing/test_pipeline.py
index 9f70e4d5c12..c8e6e7c3968 100644
--- a/sdks/python/apache_beam/testing/test_pipeline.py
+++ b/sdks/python/apache_beam/testing/test_pipeline.py
@@ -70,7 +70,8 @@ class TestPipeline(Pipeline):
       argv=None,
       is_integration_test=False,
       blocking=True,
-      additional_pipeline_args=None):
+      additional_pipeline_args=None,
+      display_data=None):
     """Initialize a pipeline object for test.
 
     Args:
@@ -93,6 +94,8 @@ class TestPipeline(Pipeline):
         completed.
       additional_pipeline_args (List[str]): additional pipeline arguments to be
         included when construction the pipeline options object.
+      display_data (Dict[str, Any]): a dictionary of static data associated
+        with this pipeline that can be displayed when it runs.
 
     Raises:
       ValueError: if either the runner or options argument is not
@@ -106,7 +109,7 @@ class TestPipeline(Pipeline):
     self.blocking = blocking
     if options is None:
       options = PipelineOptions(self.options_list)
-    super().__init__(runner, options)
+    super().__init__(runner, options, display_data=display_data)
 
   def run(self, test_runner_api=True):
     result = super().run(
diff --git a/sdks/python/apache_beam/yaml/examples/README.md b/sdks/python/apache_beam/yaml/examples/README.md
new file mode 100644
index 00000000000..0d76f41bef6
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/README.md
@@ -0,0 +1,67 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+
+# Examples Catalog
+
+<!-- TOC -->
+* [Examples Catalog](#examples-catalog)
+  * [Wordcount](#wordcount)
+  * [Transforms](#transforms)
+    * [Element-wise](#element-wise)
+    * [Aggregation](#aggregation)
+<!-- TOC -->
+
+This module contains a series of Beam YAML code samples that can be run using
+the command:
+```
+python -m apache_beam.yaml.main --pipeline_spec_file=/path/to/example.yaml
+```
+
+## Wordcount
+A good starting place is the [Wordcount](wordcount_minimal.yaml) example under
+the root example directory.
+This example reads in a text file, splits the text on each word, groups by each
+word, and counts the occurrence of each word. This is a classic example used in
+the other SDK's and shows off many of the functionalities of Beam YAML.
+
+## Transforms
+
+Examples in this directory show off the various built-in transforms of the Beam
+YAML framework.
+
+### Element-wise
+These examples leverage the built-in mapping transforms including `MapToFields`,
+`Filter` and `Explode`. More information can be found about mapping transforms
+[here](https://beam.apache.org/documentation/sdks/yaml-udf/).
+
+### Aggregation
+These examples leverage the built-in `Combine` transform for performing simple
+aggregations including sum, mean, count, etc.
+
+These examples are experimental and require that
+`yaml_experimental_features: Combine` be specified under the `options` tag, or
+by passing `--yaml_experimental_features=Combine` to the command to run the
+pipeline. i.e.
+```
+python -m apache_beam.yaml.main \
+  --pipeline_spec_file=/path/to/example.yaml \
+  --yaml_experimental_features=Combine
+```
+More information can be found about aggregation transforms
+[here](https://beam.apache.org/documentation/sdks/yaml-combine/).
\ No newline at end of file
diff --git a/sdks/python/apache_beam/yaml/examples/__init__.py b/sdks/python/apache_beam/yaml/examples/__init__.py
new file mode 100644
index 00000000000..cce3acad34a
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/sdks/python/apache_beam/yaml/examples/examples_test.py b/sdks/python/apache_beam/yaml/examples/examples_test.py
new file mode 100644
index 00000000000..e084f710a7c
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/examples_test.py
@@ -0,0 +1,154 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pytype: skip-file
+import glob
+import logging
+import os
+import random
+import unittest
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+from unittest import mock
+
+import yaml
+
+import apache_beam as beam
+from apache_beam import PCollection
+from apache_beam.examples.snippets.util import assert_matches_stdout
+from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.yaml import yaml_transform
+from apache_beam.yaml.readme_test import TestEnvironment
+from apache_beam.yaml.readme_test import replace_recursive
+
+
+def check_output(expected: List[str]):
+  def _check_inner(actual: PCollection[str]):
+    formatted_actual = actual | beam.Map(
+        lambda row: str(beam.Row(**row._asdict())))
+    assert_matches_stdout(formatted_actual, expected)
+
+  return _check_inner
+
+
+def create_test_method(
+    pipeline_spec_file: str,
+    custom_preprocessor: Optional[Callable[..., Union[Dict, List]]] = None):
+  @mock.patch('apache_beam.Pipeline', TestPipeline)
+  def test_yaml_example(self):
+    with open(pipeline_spec_file, encoding="utf-8") as f:
+      lines = f.readlines()
+    expected_key = '# Expected:\n'
+    if expected_key in lines:
+      expected = lines[lines.index('# Expected:\n') + 1:]
+    else:
+      raise ValueError(
+          f"Missing '# Expected:' tag in example file '{pipeline_spec_file}'")
+    for i, line in enumerate(expected):
+      expected[i] = line.replace('#  ', '').replace('\n', '')
+    pipeline_spec = yaml.load(
+        ''.join(lines), Loader=yaml_transform.SafeLineLoader)
+
+    with TestEnvironment() as env:
+      if custom_preprocessor:
+        pipeline_spec = custom_preprocessor(pipeline_spec, expected, env)
+      with beam.Pipeline(options=PipelineOptions(
+          pickle_library='cloudpickle',
+          **yaml_transform.SafeLineLoader.strip_metadata(pipeline_spec.get(
+              'options', {})))) as p:
+        actual = yaml_transform.expand_pipeline(p, pipeline_spec)
+        check_output(expected)(actual)
+
+  return test_yaml_example
+
+
+class YamlExamplesTestSuite:
+  _test_preprocessor: Dict[str, Callable[..., Union[Dict, List]]] = {}
+
+  def __init__(self, name: str, path: str):
+    self._test_suite = self.create_test_suite(name, path)
+
+  def run(self):
+    return self._test_suite
+
+  @classmethod
+  def parse_test_methods(cls, path: str):
+    files = glob.glob(path)
+    if not files and os.path.exists(path) and os.path.isfile(path):
+      files = [path]
+    for file in files:
+      test_name = f'test_{file.split(os.sep)[-1].replace(".", "_")}'
+      custom_preprocessor = cls._test_preprocessor.get(test_name, None)
+      yield test_name, create_test_method(file, custom_preprocessor)
+
+  @classmethod
+  def create_test_suite(cls, name: str, path: str):
+    return type(name, (unittest.TestCase, ), dict(cls.parse_test_methods(path)))
+
+  @classmethod
+  def register_test_preprocessor(cls, test_name: str):
+    def apply(preprocessor):
+      cls._test_preprocessor[test_name] = preprocessor
+      return preprocessor
+
+    return apply
+
+
+@YamlExamplesTestSuite.register_test_preprocessor('test_wordcount_minimal_yaml')
+def _wordcount_test_preprocessor(
+    test_spec: str, expected: List[str], env: TestEnvironment):
+  all_words = []
+  for element in expected:
+    word = element.split('=')[1].split(',')[0].replace("'", '')
+    count = int(element.split('=')[2].replace(')', ''))
+    all_words += [word] * count
+  random.shuffle(all_words)
+
+  lines = []
+  while all_words:
+    line_length = random.randint(1, min(10, len(all_words)))
+    line = " ".join(
+        all_words.pop(random.randrange(len(all_words)))
+        for _ in range(line_length))
+    lines.append(line)
+
+  return replace_recursive(
+      test_spec,
+      'ReadFromText',
+      'path',
+      env.input_file('kinglear.txt', '\n'.join(lines)))
+
+
+YAML_DOCS_DIR = os.path.join(os.path.dirname(__file__))
+ExamplesTest = YamlExamplesTestSuite(
+    'ExamplesTest', os.path.join(YAML_DOCS_DIR, '*.yaml')).run()
+
+ElementWiseTest = YamlExamplesTestSuite(
+    'ElementwiseExamplesTest',
+    os.path.join(YAML_DOCS_DIR, 'transforms/elementwise/*.yaml')).run()
+
+AggregationTest = YamlExamplesTestSuite(
+    'AggregationExamplesTest',
+    os.path.join(YAML_DOCS_DIR, 'transforms/aggregation/*.yaml')).run()
+
+if __name__ == '__main__':
+  logging.getLogger().setLevel(logging.INFO)
+  unittest.main()
diff --git a/sdks/python/apache_beam/yaml/examples/regex_matches.yaml b/sdks/python/apache_beam/yaml/examples/regex_matches.yaml
new file mode 100644
index 00000000000..176813cef6b
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/regex_matches.yaml
@@ -0,0 +1,62 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# pytype: skip-file
+
+# This pipline creates a series of {plant: description} key pairs, matches all
+# elements to a valid regex, filters out non-matching entries, then logs the
+# output.
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Garden plants
+      config:
+        elements:
+          - plant: 'πŸ“, Strawberry, perennial'
+          - plant:  'πŸ₯•, Carrot, biennial ignoring trailing words'
+          - plant:  'πŸ†, Eggplant, perennial'
+          - plant:  'πŸ…, Tomato, annual'
+          - plant:  'πŸ₯”, Potato, perennial'
+          - plant:  '# 🍌, invalid, format'
+          - plant:  'invalid, πŸ‰, format'
+    - type: MapToFields
+      name: Parse plants
+      config:
+        language: python
+        fields:
+          plant:
+            callable: |
+              import re
+              def regex_filter(row):
+                match = re.match("(?P<icon>[^\s,]+), *(\w+), *(\w+)", row.plant)
+                return match.group(0) if match else match
+                
+    # Filters out None values produced by values that don't match regex
+    - type: Filter
+      config:
+        language: python
+        keep: plant
+    - type: LogForTesting
+      
+# Expected:
+#  Row(plant='πŸ“, Strawberry, perennial')
+#  Row(plant='πŸ₯•, Carrot, biennial')
+#  Row(plant='πŸ†, Eggplant, perennial')
+#  Row(plant='πŸ…, Tomato, annual')
+#  Row(plant='πŸ₯”, Potato, perennial')
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_count_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_count_minimal.yaml
new file mode 100644
index 00000000000..b746e55e344
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_count_minimal.yaml
@@ -0,0 +1,62 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce names
+      config:
+        elements:
+          - season: 'spring'
+            produce: 'πŸ“'
+          - season: 'spring'
+            produce: 'πŸ₯•'
+          - season: 'summer'
+            produce: 'πŸ₯•'
+          - season: 'fall'
+            produce: 'πŸ₯•'
+          - season: 'spring'
+            produce: 'πŸ†'
+          - season: 'winter'
+            produce: 'πŸ†'
+          - season: 'spring'
+            produce: 'πŸ…'
+          - season: 'summer'
+            produce: 'πŸ…'
+          - season: 'fall'
+            produce: 'πŸ…'
+          - season: 'summer'
+            produce: '🌽'
+    - type: Combine
+      name: Shortest names per key
+      config:
+        language: python
+        group_by: season
+        combine:
+          produce: count
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(season='spring', produce=4)
+#  Row(season='summer', produce=3)
+#  Row(season='fall', produce=2)
+#  Row(season='winter', produce=1)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_max_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_max_minimal.yaml
new file mode 100644
index 00000000000..2588897a717
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_max_minimal.yaml
@@ -0,0 +1,53 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce
+      config:
+        elements:
+          - produce: 'πŸ₯•'
+            amount: 3
+          - produce: 'πŸ₯•'
+            amount: 2
+          - produce: 'πŸ†'
+            amount: 1
+          - produce: 'πŸ…'
+            amount: 4
+          - produce: 'πŸ…'
+            amount: 5
+          - produce: 'πŸ…'
+            amount: 3
+    - type: Combine
+      name: Get max value per key
+      config:
+        language: python
+        group_by: produce
+        combine:
+          amount: max
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(produce='πŸ₯•', amount=3)
+#  Row(produce='πŸ†', amount=1)
+#  Row(produce='πŸ…', amount=5)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_mean_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_mean_minimal.yaml
new file mode 100644
index 00000000000..b00b7e817ab
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_mean_minimal.yaml
@@ -0,0 +1,53 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce
+      config:
+        elements:
+          - produce: 'πŸ₯•'
+            amount: 3
+          - produce: 'πŸ₯•'
+            amount: 2
+          - produce: 'πŸ†'
+            amount: 1
+          - produce: 'πŸ…'
+            amount: 4
+          - produce: 'πŸ…'
+            amount: 5
+          - produce: 'πŸ…'
+            amount: 3
+    - type: Combine
+      name: Get mean value per key
+      config:
+        language: python
+        group_by: produce
+        combine:
+          amount: mean
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(produce='πŸ₯•', amount=2.5)
+#  Row(produce='πŸ†', amount=1.0)
+#  Row(produce='πŸ…', amount=4.0)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_min_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_min_minimal.yaml
new file mode 100644
index 00000000000..1e05fd3f755
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_min_minimal.yaml
@@ -0,0 +1,53 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce
+      config:
+        elements:
+          - produce: 'πŸ₯•'
+            amount: 3
+          - produce: 'πŸ₯•'
+            amount: 2
+          - produce: 'πŸ†'
+            amount: 1
+          - produce: 'πŸ…'
+            amount: 4
+          - produce: 'πŸ…'
+            amount: 5
+          - produce: 'πŸ…'
+            amount: 3
+    - type: Combine
+      name: Get min value per key
+      config:
+        language: python
+        group_by: produce
+        combine:
+          amount: min
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(produce='πŸ₯•', amount=2)
+#  Row(produce='πŸ†', amount=1)
+#  Row(produce='πŸ…', amount=3)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_multiple_aggregations.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_multiple_aggregations.yaml
new file mode 100644
index 00000000000..1263ac21fac
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_multiple_aggregations.yaml
@@ -0,0 +1,83 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce
+      config:
+        elements:
+          - recipe: 'pie'
+            fruit: 'raspberry'
+            quantity: 1
+            unit_price: 3.50
+          - recipe: 'pie'
+            fruit: 'blackberry'
+            quantity: 1
+            unit_price: 4.00
+          - recipe: 'pie'
+            fruit: 'blueberry'
+            quantity: 1
+            unit_price: 2.00
+          - recipe: 'muffin'
+            fruit: 'blueberry'
+            quantity: 2
+            unit_price: 2.00
+          - recipe: 'muffin'
+            fruit: 'banana'
+            quantity: 3
+            unit_price: 1.00
+    # Simulates global GroupBy by creating global key
+    - type: MapToFields
+      name: Add global key
+      config:
+        append: true
+        language: python
+        fields:
+          global_key: '0'
+    - type: Combine
+      name: Get multiple aggregations per key
+      config:
+        language: python
+        group_by: global_key
+        combine:
+          min_price: 
+            value: unit_price
+            fn: min
+          mean_price: 
+            value: unit_price
+            fn: mean
+          max_price: 
+            value: unit_price
+            fn: max
+    # Removes unwanted global key
+    - type: MapToFields
+      name: Remove global key
+      config:
+        fields:
+          min_price: min_price
+          mean_price: mean_price
+          max_price: max_price
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(min_price=1.0, mean_price=2.5, max_price=4.0)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum.yaml
new file mode 100644
index 00000000000..77cd1529b58
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum.yaml
@@ -0,0 +1,67 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce
+      config:
+        elements:
+          - recipe: 'pie'
+            fruit: 'raspberry'
+            quantity: 1
+            unit_price: 3.50
+          - recipe: 'pie'
+            fruit: 'blackberry'
+            quantity: 1
+            unit_price: 4.00
+          - recipe: 'pie'
+            fruit: 'blueberry'
+            quantity: 1
+            unit_price: 2.00
+          - recipe: 'muffin'
+            fruit: 'blueberry'
+            quantity: 2
+            unit_price: 3.00
+          - recipe: 'muffin'
+            fruit: 'banana'
+            quantity: 3
+            unit_price: 1.00
+    - type: Combine
+      name: Sum values per key
+      config:
+        language: python
+        group_by: fruit
+        combine:
+          total_quantity: 
+            value: quantity
+            fn: sum
+          mean_price:
+            value: unit_price
+            fn: mean
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(fruit='raspberry', total_quantity=1, mean_price=3.5)
+#  Row(fruit='blackberry', total_quantity=1, mean_price=4.0)
+#  Row(fruit='blueberry', total_quantity=3, mean_price=2.5)
+#  Row(fruit='banana', total_quantity=3, mean_price=1.0)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum_minimal.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum_minimal.yaml
new file mode 100644
index 00000000000..905a30d13f0
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/combine_sum_minimal.yaml
@@ -0,0 +1,53 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce
+      config:
+        elements:
+          - produce: 'πŸ₯•'
+            amount: 3
+          - produce: 'πŸ₯•'
+            amount: 2
+          - produce: 'πŸ†'
+            amount: 1
+          - produce: 'πŸ…'
+            amount: 4
+          - produce: 'πŸ…'
+            amount: 5
+          - produce: 'πŸ…'
+            amount: 3
+    - type: Combine
+      name: Count elements per key
+      config:
+        language: python
+        group_by: produce
+        combine:
+          amount: sum
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(produce='πŸ₯•', amount=5)
+#  Row(produce='πŸ†', amount=1)
+#  Row(produce='πŸ…', amount=12)
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/group_into_batches.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/group_into_batches.yaml
new file mode 100644
index 00000000000..a956ea604c6
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/group_into_batches.yaml
@@ -0,0 +1,67 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce
+      config:
+        elements:
+          - season: 'spring'
+            produce: 'πŸ“'
+          - season: 'spring'
+            produce: 'πŸ₯•'
+          - season: 'spring'
+            produce: 'πŸ†'
+          - season: 'spring'
+            produce: 'πŸ…'
+          - season: 'summer'
+            produce: 'πŸ₯•'
+          - season: 'summer'
+            produce: 'πŸ…'
+          - season: 'summer'
+            produce: '🌽'
+          - season: 'fall'
+            produce: 'πŸ₯•'
+          - season: 'fall'
+            produce: 'πŸ…'
+          - season: 'winter'
+            produce: 'πŸ†'
+    - type: Combine
+      name: Group into batches
+      config:
+        language: python
+        group_by: season
+        combine:
+          produce:
+            value: produce
+            fn:
+              type: 'apache_beam.transforms.combiners.TopCombineFn'
+              config:
+                n: 3
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(season='spring', produce=['πŸ₯•', 'πŸ“', 'πŸ†'])
+#  Row(season='summer', produce=['πŸ₯•', 'πŸ…', '🌽'])
+#  Row(season='fall', produce=['πŸ₯•', 'πŸ…'])
+#  Row(season='winter', produce=['πŸ†'])
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_largest_per_key.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_largest_per_key.yaml
new file mode 100644
index 00000000000..c35a5da1bd4
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_largest_per_key.yaml
@@ -0,0 +1,57 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce
+      config:
+        elements:
+          - produce: 'πŸ₯•'
+            amount: 3
+          - produce: 'πŸ₯•'
+            amount: 2
+          - produce: 'πŸ†'
+            amount: 1
+          - produce: 'πŸ…'
+            amount: 4
+          - produce: 'πŸ…'
+            amount: 5
+          - produce: 'πŸ…'
+            amount: 3
+    - type: Combine
+      config:
+        language: python
+        group_by: produce
+        combine:
+          biggest:
+            value: amount
+            fn:
+              type: 'apache_beam.transforms.combiners.TopCombineFn'
+              config:
+                n: 2
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(produce='πŸ₯•', biggest=[3, 2])
+#  Row(produce='πŸ†', biggest=[1])
+#  Row(produce='πŸ…', biggest=[5, 4])
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_smallest_per_key.yaml b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_smallest_per_key.yaml
new file mode 100644
index 00000000000..ebd7c0c34fe
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/aggregation/top_smallest_per_key.yaml
@@ -0,0 +1,59 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Create produce
+      config:
+        elements:
+          - produce: 'πŸ₯•'
+            amount: 3
+          - produce: 'πŸ₯•'
+            amount: 2
+          - produce: 'πŸ†'
+            amount: 1
+          - produce: 'πŸ…'
+            amount: 4
+          - produce: 'πŸ…'
+            amount: 5
+          - produce: 'πŸ…'
+            amount: 3
+    - type: Combine
+      name: Smallest N values per key
+      config:
+        language: python
+        group_by: produce
+        combine:
+          smallest:
+            value: amount
+            fn:
+              type: 'apache_beam.transforms.combiners.TopCombineFn'
+              config:
+                n: 2
+                reverse: true
+    - type: LogForTesting
+
+options:
+  yaml_experimental_features: Combine
+
+# Expected:
+#  Row(produce='πŸ₯•', smallest=[2, 3])
+#  Row(produce='πŸ†', smallest=[1])
+#  Row(produce='πŸ…', smallest=[3, 4])
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/elementwise/explode.yaml b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/explode.yaml
new file mode 100644
index 00000000000..70432ee7735
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/explode.yaml
@@ -0,0 +1,41 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the 'License'); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Gardening plants
+      config:
+        elements:
+          - produce: ['πŸ“Strawberry', 'πŸ₯”Potato']
+            season: 'spring'
+          - produce: ['πŸ₯•Carrot', 'πŸ†Eggplant', 'πŸ…Tomato']
+            season: 'summer'
+    - type: Explode
+      name: Flatten lists
+      config:
+        fields: [produce]
+    - type: LogForTesting
+
+# Expected:
+#  Row(produce='πŸ“Strawberry', season='spring')
+#  Row(produce='πŸ₯•Carrot', season='summer')
+#  Row(produce='πŸ†Eggplant', season='summer')
+#  Row(produce='πŸ…Tomato', season='summer')
+#  Row(produce='πŸ₯”Potato', season='spring')
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/elementwise/filter_callable.yaml b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/filter_callable.yaml
new file mode 100644
index 00000000000..f66f672fd59
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/filter_callable.yaml
@@ -0,0 +1,52 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Gardening plants
+      config:
+        elements:
+          - 'icon': 'πŸ“'
+            'name': 'Strawberry'
+            'duration': 'perennial'
+          - 'icon': 'πŸ₯•'
+            'name': 'Carrot'
+            'duration': 'biennial'
+          - 'icon': 'πŸ†'
+            'name': 'Eggplant'
+            'duration': 'perennial'
+          - 'icon': 'πŸ…'
+            'name': 'Tomato'
+            'duration': 'annual'
+          - 'icon': 'πŸ₯”'
+            'name': 'Potato'
+            'duration': 'perennial'
+    - type: Filter
+      name: Filter perennials
+      config:
+        language: python
+        keep:
+          callable: "lambda plant: plant.duration == 'perennial'"
+    - type: LogForTesting
+
+# Expected:
+#  Row(icon='πŸ“', name='Strawberry', duration='perennial')
+#  Row(icon='πŸ†', name='Eggplant', duration='perennial')
+#  Row(icon='πŸ₯”', name='Potato', duration='perennial')
diff --git a/sdks/python/apache_beam/yaml/examples/transforms/elementwise/map_to_fields_callable.yaml b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/map_to_fields_callable.yaml
new file mode 100644
index 00000000000..a6468b8829f
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/transforms/elementwise/map_to_fields_callable.yaml
@@ -0,0 +1,45 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pipeline:
+  type: chain
+  transforms:
+    - type: Create
+      name: Gardening plants
+      config:
+        elements:
+          - '# πŸ“Strawberry'
+          - '# πŸ₯•Carrot'
+          - '# πŸ†Eggplant'
+          - '# πŸ…Tomato'
+          - '# πŸ₯”Potato'
+    - type: MapToFields
+      name: Strip header
+      config:
+        language: python
+        fields:
+          element:
+            callable: "lambda row: row.element.strip('# \\n')"
+    - type: LogForTesting
+      
+# Expected:
+#  Row(element='πŸ“Strawberry')
+#  Row(element='πŸ₯•Carrot')
+#  Row(element='πŸ†Eggplant')
+#  Row(element='πŸ…Tomato')
+#  Row(element='πŸ₯”Potato')
diff --git a/sdks/python/apache_beam/yaml/examples/wordcount_minimal.yaml b/sdks/python/apache_beam/yaml/examples/wordcount_minimal.yaml
new file mode 100644
index 00000000000..044db2790ac
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/examples/wordcount_minimal.yaml
@@ -0,0 +1,87 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the#  Row(word='License'); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an#  Row(word='AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This examples reads from a public file stores on Google Cloud. This
+# requires authenticating with Google Cloud, or setting the file in
+#`ReadFromText` to a local file.
+#
+# To set up Application Default Credentials,
+# see https://cloud.google.com/docs/authentication/external/set-up-adc for more
+# information
+#
+# This pipeline reads in a text file, counts distinct words found in the text, 
+# then logs a row containing each word and its count.
+pipeline:
+  type: chain
+  transforms:
+
+    # Read text file into a collection of rows, each with one field, "line"
+    - type: ReadFromText
+      config:
+        path: gs://dataflow-samples/shakespeare/kinglear.txt
+
+    # Split line field in each row into list of words
+    - type: MapToFields
+      config:
+        language: python
+        fields:
+          words:
+            callable: |
+              import re
+              def my_mapping(row):
+                return re.findall(r"[A-Za-z\']+", row.line.lower())
+
+    # Explode each list of words into separate rows
+    - type: Explode
+      config:
+        fields: words
+
+    # Since each word is now distinct row, rename field to "word"
+    - type: MapToFields
+      config:
+        fields:
+          word: words
+
+    # Group by distinct words in the collection and add field "count" that
+    # contains number of instances, or count, for each word in the collection.
+    - type: Combine
+      config:
+        language: python
+        group_by: word
+        combine:
+          count:
+            value: word
+            fn: count
+
+    # Log out results
+    - type: LogForTesting
+        
+options:
+  yaml_experimental_features: Combine
+  
+# Expected:
+#  Row(word='king', count=311)
+#  Row(word='lear', count=253)
+#  Row(word='dramatis', count=1)
+#  Row(word='personae', count=1)
+#  Row(word='of', count=483)
+#  Row(word='britain', count=2)
+#  Row(word='france', count=32)
+#  Row(word='duke', count=26)
+#  Row(word='burgundy', count=20)
+#  Row(word='cornwall', count=75)
diff --git a/sdks/python/apache_beam/yaml/readme_test.py b/sdks/python/apache_beam/yaml/readme_test.py
index 9a0e4454611..4ca60e6176b 100644
--- a/sdks/python/apache_beam/yaml/readme_test.py
+++ b/sdks/python/apache_beam/yaml/readme_test.py
@@ -191,10 +191,10 @@ def replace_recursive(spec, transform_type, arg_name, arg_value):
 
 
 def create_test_method(test_type, test_name, test_yaml):
-  test_yaml = test_yaml.replace(
-      'pkg.module.', 'apache_beam.yaml.readme_test._Fakes.')
   test_yaml = test_yaml.replace(
       'apache_beam.pkg.module.', 'apache_beam.yaml.readme_test._Fakes.')
+  test_yaml = test_yaml.replace(
+      'pkg.module.', 'apache_beam.yaml.readme_test._Fakes.')
 
   def test(self):
     with TestEnvironment() as env: