You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tv...@apache.org on 2021/05/24 19:50:37 UTC
[beam] branch master updated: [BEAM-10144] update pipeline options
snippets (#14738)
This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8a1f17e [BEAM-10144] update pipeline options snippets (#14738)
8a1f17e is described below
commit 8a1f17eeb86abe6c3266e15cf3a312175a94cbab
Author: David Cavazos <dc...@google.com>
AuthorDate: Mon May 24 12:49:35 2021 -0700
[BEAM-10144] update pipeline options snippets (#14738)
---
.../apache_beam/examples/snippets/snippets.py | 580 +++++++++++----------
.../apache_beam/examples/snippets/snippets_test.py | 42 +-
2 files changed, 326 insertions(+), 296 deletions(-)
diff --git a/sdks/python/apache_beam/examples/snippets/snippets.py b/sdks/python/apache_beam/examples/snippets/snippets.py
index 82d3ecc..78814cd 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets.py
@@ -36,6 +36,8 @@ import base64
import json
from decimal import Decimal
+import mock
+
import apache_beam as beam
from apache_beam.io import iobase
from apache_beam.io.range_trackers import OffsetRangeTracker
@@ -88,6 +90,7 @@ class SnippetUtils(object):
transform_node.transform.fn.file_to_write = self.renames['write']
+@mock.patch('apache_beam.Pipeline', TestPipeline)
def construct_pipeline(renames):
"""A reverse words snippet as an example for constructing a pipeline."""
import re
@@ -97,10 +100,12 @@ def construct_pipeline(renames):
# Unresolved reference in ReverseWords class
import apache_beam as beam
- class ReverseWords(beam.PTransform):
+ @beam.ptransform_fn
+ @beam.typehints.with_input_types(str)
+ @beam.typehints.with_output_types(str)
+ def ReverseWords(pcoll):
"""A PTransform that reverses individual elements in a PCollection."""
- def expand(self, pcoll):
- return pcoll | beam.Map(lambda e: e[::-1])
+ return pcoll | beam.Map(lambda word: word[::-1])
def filter_words(unused_x):
"""Pass through filter to select everything."""
@@ -108,82 +113,66 @@ def construct_pipeline(renames):
# [START pipelines_constructing_creating]
import apache_beam as beam
- from apache_beam.options.pipeline_options import PipelineOptions
- with beam.Pipeline(options=PipelineOptions()) as p:
+ with beam.Pipeline() as pipeline:
pass # build your pipeline here
# [END pipelines_constructing_creating]
- with TestPipeline() as p: # Use TestPipeline for testing.
- # pylint: disable=line-too-long
-
- # [START pipelines_constructing_reading]
- lines = p | 'ReadMyFile' >> beam.io.ReadFromText(
- 'gs://some/inputData.txt')
- # [END pipelines_constructing_reading]
+ # [START pipelines_constructing_reading]
+ lines = pipeline | 'ReadMyFile' >> beam.io.ReadFromText(
+ 'gs://some/inputData.txt')
+ # [END pipelines_constructing_reading]
- # [START pipelines_constructing_applying]
- words = lines | beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
- reversed_words = words | ReverseWords()
- # [END pipelines_constructing_applying]
+ # [START pipelines_constructing_applying]
+ words = lines | beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
+ reversed_words = words | ReverseWords()
+ # [END pipelines_constructing_applying]
- # [START pipelines_constructing_writing]
- filtered_words = reversed_words | 'FilterWords' >> beam.Filter(
- filter_words)
- filtered_words | 'WriteMyFile' >> beam.io.WriteToText(
- 'gs://some/outputData.txt')
- # [END pipelines_constructing_writing]
+ # [START pipelines_constructing_writing]
+ filtered_words = reversed_words | 'FilterWords' >> beam.Filter(filter_words)
+ filtered_words | 'WriteMyFile' >> beam.io.WriteToText(
+ 'gs://some/outputData.txt')
+ # [END pipelines_constructing_writing]
- p.visit(SnippetUtils.RenameFiles(renames))
+ pipeline.visit(SnippetUtils.RenameFiles(renames))
-def model_pipelines(argv):
+def model_pipelines():
"""A wordcount snippet as a simple pipeline example."""
# [START model_pipelines]
+ import argparse
import re
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
- class MyOptions(PipelineOptions):
- @classmethod
- def _add_argparse_args(cls, parser):
- parser.add_argument(
- '--input',
- dest='input',
- default='gs://dataflow-samples/shakespeare/kinglear'
- '.txt',
- help='Input file to process.')
- parser.add_argument(
- '--output',
- dest='output',
- required=True,
- help='Output file to write results to.')
-
- pipeline_options = PipelineOptions(argv)
- my_options = pipeline_options.view_as(MyOptions)
-
- with beam.Pipeline(options=pipeline_options) as p:
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--input-file',
+ default='gs://dataflow-samples/shakespeare/kinglear.txt',
+ help='The file path for the input text to process.')
+ parser.add_argument(
+ '--output-path', required=True, help='The path prefix for output files.')
+ args, beam_args = parser.parse_known_args()
+ beam_options = PipelineOptions(beam_args)
+ with beam.Pipeline(options=beam_options) as pipeline:
(
- p
- | beam.io.ReadFromText(my_options.input)
+ pipeline
+ | beam.io.ReadFromText(args.input_file)
| beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
| beam.Map(lambda x: (x, 1))
| beam.combiners.Count.PerKey()
- | beam.io.WriteToText(my_options.output))
+ | beam.io.WriteToText(args.output_path))
# [END model_pipelines]
-def model_pcollection(argv):
+def model_pcollection(output_path):
"""Creating a PCollection from data in local memory."""
# [START model_pcollection]
import apache_beam as beam
- from apache_beam.options.pipeline_options import PipelineOptions
- # argv = None # if None, uses sys.argv
- pipeline_options = PipelineOptions(argv)
- with beam.Pipeline(options=pipeline_options) as pipeline:
+ with beam.Pipeline() as pipeline:
lines = (
pipeline
| beam.Create([
@@ -192,30 +181,18 @@ def model_pcollection(argv):
'The slings and arrows of outrageous fortune, ',
'Or to take arms against a sea of troubles, ',
]))
-
# [END model_pcollection]
- class MyOptions(PipelineOptions):
- @classmethod
- def _add_argparse_args(cls, parser):
- parser.add_argument(
- '--output',
- dest='output',
- required=True,
- help='Output file to write results to.')
+ lines | beam.io.WriteToText(output_path)
- my_options = pipeline_options.view_as(MyOptions)
- lines | beam.io.WriteToText(my_options.output)
-
-def pipeline_options_remote(argv):
+def pipeline_options_remote():
"""Creating a Pipeline using a PipelineOptions object for remote execution."""
# [START pipeline_options_create]
from apache_beam.options.pipeline_options import PipelineOptions
- options = PipelineOptions(flags=argv)
-
+ beam_options = PipelineOptions()
# [END pipeline_options_create]
# [START pipeline_options_define_custom]
@@ -224,46 +201,54 @@ def pipeline_options_remote(argv):
class MyOptions(PipelineOptions):
@classmethod
def _add_argparse_args(cls, parser):
- parser.add_argument('--input')
- parser.add_argument('--output')
+ parser.add_argument('--input-file')
+ parser.add_argument('--output-path')
# [END pipeline_options_define_custom]
- # [START pipeline_options_dataflow_service]
- import apache_beam as beam
- from apache_beam.options.pipeline_options import PipelineOptions
-
- # Create and set your PipelineOptions.
- # For Cloud execution, specify DataflowRunner and set the Cloud Platform
- # project, job name, temporary files location, and region.
- # For more information about regions, check:
- # https://cloud.google.com/dataflow/docs/concepts/regional-endpoints
- options = PipelineOptions(
- flags=argv,
- runner='DataflowRunner',
- project='my-project-id',
- job_name='unique-job-name',
- temp_location='gs://my-bucket/temp',
- region='us-central1')
-
- # Create the Pipeline with the specified options.
- # with beam.Pipeline(options=options) as pipeline:
- # pass # build your pipeline here.
- # [END pipeline_options_dataflow_service]
-
- my_options = options.view_as(MyOptions)
-
- with TestPipeline() as p: # Use TestPipeline for testing.
- lines = p | beam.io.ReadFromText(my_options.input)
- lines | beam.io.WriteToText(my_options.output)
-
-
-def pipeline_options_local(argv):
+ @mock.patch('apache_beam.Pipeline')
+ def dataflow_options(mock_pipeline):
+ # [START pipeline_options_dataflow_service]
+ import argparse
+
+ import apache_beam as beam
+ from apache_beam.options.pipeline_options import PipelineOptions
+
+ parser = argparse.ArgumentParser()
+ # parser.add_argument('--my-arg', help='description')
+ args, beam_args = parser.parse_known_args()
+
+ # Create and set your PipelineOptions.
+ # For Cloud execution, specify DataflowRunner and set the Cloud Platform
+ # project, job name, temporary files location, and region.
+ # For more information about regions, check:
+ # https://cloud.google.com/dataflow/docs/concepts/regional-endpoints
+ beam_options = PipelineOptions(
+ beam_args,
+ runner='DataflowRunner',
+ project='my-project-id',
+ job_name='unique-job-name',
+ temp_location='gs://my-bucket/temp',
+ region='us-central1')
+
+ # Create the Pipeline with the specified options.
+ with beam.Pipeline(options=beam_options) as pipeline:
+ pass # build your pipeline here.
+ # [END pipeline_options_dataflow_service]
+ return beam_options
+
+ beam_options = dataflow_options()
+ args = beam_options.view_as(MyOptions)
+
+ with TestPipeline() as pipeline: # Use TestPipeline for testing.
+ lines = pipeline | beam.io.ReadFromText(args.input_file)
+ lines | beam.io.WriteToText(args.output_path)
+
+
+@mock.patch('apache_beam.Pipeline', TestPipeline)
+def pipeline_options_local():
"""Creating a Pipeline using a PipelineOptions object for local execution."""
- from apache_beam import Pipeline
- from apache_beam.options.pipeline_options import PipelineOptions
-
# [START pipeline_options_define_custom_with_help_and_default]
from apache_beam.options.pipeline_options import PipelineOptions
@@ -271,31 +256,40 @@ def pipeline_options_local(argv):
@classmethod
def _add_argparse_args(cls, parser):
parser.add_argument(
- '--input',
- help='Input for the pipeline',
- default='gs://my-bucket/input')
+ '--input-file',
+ default='gs://dataflow-samples/shakespeare/kinglear.txt',
+ help='The file path for the input text to process.')
parser.add_argument(
- '--output',
- help='Output for the pipeline',
- default='gs://my-bucket/output')
+ '--output-path',
+ required=True,
+ help='The path prefix for output files.')
# [END pipeline_options_define_custom_with_help_and_default]
# [START pipeline_options_local]
- # Create and set your Pipeline Options.
- options = PipelineOptions(flags=argv)
- my_options = options.view_as(MyOptions)
+ import argparse
+
+ import apache_beam as beam
+ from apache_beam.options.pipeline_options import PipelineOptions
+
+ parser = argparse.ArgumentParser()
+ # parser.add_argument('--my-arg')
+ args, beam_args = parser.parse_known_args()
- with Pipeline(options=options) as pipeline:
- pass # build your pipeline here.
- # [END pipeline_options_local]
+ # Create and set your Pipeline Options.
+ beam_options = PipelineOptions(beam_args)
+ args = beam_options.view_as(MyOptions)
- with TestPipeline() as p: # Use TestPipeline for testing.
- lines = p | beam.io.ReadFromText(my_options.input)
- lines | beam.io.WriteToText(my_options.output)
+ with beam.Pipeline(options=beam_options) as pipeline:
+ lines = (
+ pipeline
+ | beam.io.ReadFromText(args.input_file)
+ | beam.io.WriteToText(args.output_path))
+ # [END pipeline_options_local]
-def pipeline_options_command_line(argv):
+@mock.patch('apache_beam.Pipeline', TestPipeline)
+def pipeline_options_command_line():
"""Creating a Pipeline by passing a list of arguments."""
# [START pipeline_options_command_line]
@@ -303,17 +297,27 @@ def pipeline_options_command_line(argv):
import argparse
import apache_beam as beam
+ from apache_beam.options.pipeline_options import PipelineOptions
+ # For more details on how to use argparse, take a look at:
+ # https://docs.python.org/3/library/argparse.html
parser = argparse.ArgumentParser()
- parser.add_argument('--input')
- parser.add_argument('--output')
- args, beam_args = parser.parse_known_args(argv)
+ parser.add_argument(
+ '--input-file',
+ default='gs://dataflow-samples/shakespeare/kinglear.txt',
+ help='The file path for the input text to process.')
+ parser.add_argument(
+ '--output-path', required=True, help='The path prefix for output files.')
+ args, beam_args = parser.parse_known_args()
# Create the Pipeline with remaining arguments.
- with beam.Pipeline(argv=beam_args) as pipeline:
- lines = pipeline | 'Read files' >> beam.io.ReadFromText(args.input)
- lines | 'Write files' >> beam.io.WriteToText(args.output)
- # [END pipeline_options_command_line]
+ beam_options = PipelineOptions(beam_args)
+ with beam.Pipeline(options=beam_options) as pipeline:
+ lines = (
+ pipeline
+ | 'Read files' >> beam.io.ReadFromText(args.input_file)
+ | 'Write files' >> beam.io.WriteToText(args.output_path))
+ # [END pipeline_options_command_line]
def pipeline_logging(lines, output):
@@ -339,32 +343,20 @@ def pipeline_logging(lines, output):
# Remaining WordCount example code ...
# [END pipeline_logging]
- with TestPipeline() as p: # Use TestPipeline for testing.
+ with TestPipeline() as pipeline: # Use TestPipeline for testing.
(
- p
+ pipeline
| beam.Create(lines)
| beam.ParDo(ExtractWordsFn())
| beam.io.WriteToText(output))
-def pipeline_monitoring(renames):
+def pipeline_monitoring():
"""Using monitoring interface snippets."""
+ import argparse
import re
import apache_beam as beam
- from apache_beam.options.pipeline_options import PipelineOptions
-
- class WordCountOptions(PipelineOptions):
- @classmethod
- def _add_argparse_args(cls, parser):
- parser.add_argument(
- '--input',
- help='Input for the pipeline',
- default='gs://my-bucket/input')
- parser.add_argument(
- '--output',
- help='output for the pipeline',
- default='gs://my-bucket/output')
class ExtractWordsFn(beam.DoFn):
def process(self, element):
@@ -379,68 +371,83 @@ def pipeline_monitoring(renames):
# [START pipeline_monitoring_composite]
# The CountWords Composite Transform inside the WordCount pipeline.
- class CountWords(beam.PTransform):
- def expand(self, pcoll):
- return (
- pcoll
- # Convert lines of text into individual words.
- | 'ExtractWords' >> beam.ParDo(ExtractWordsFn())
- # Count the number of times each word occurs.
- | beam.combiners.Count.PerElement()
- # Format each word and count into a printable string.
- | 'FormatCounts' >> beam.ParDo(FormatCountsFn()))
+ @beam.ptransform_fn
+ def CountWords(pcoll):
+ return (
+ pcoll
+ # Convert lines of text into individual words.
+ | 'ExtractWords' >> beam.ParDo(ExtractWordsFn())
+ # Count the number of times each word occurs.
+ | beam.combiners.Count.PerElement()
+ # Format each word and count into a printable string.
+ | 'FormatCounts' >> beam.ParDo(FormatCountsFn()))
# [END pipeline_monitoring_composite]
- pipeline_options = PipelineOptions()
- options = pipeline_options.view_as(WordCountOptions)
- with TestPipeline() as p: # Use TestPipeline for testing.
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--input-file',
+ default='gs://dataflow-samples/shakespeare/kinglear.txt',
+ help='The file path for the input text to process.')
+ parser.add_argument(
+ '--output-path', required=True, help='The path prefix for output files.')
+ args, _ = parser.parse_known_args()
+
+ with TestPipeline() as pipeline: # Use TestPipeline for testing.
# [START pipeline_monitoring_execution]
(
- p
+ pipeline
# Read the lines of the input text.
- | 'ReadLines' >> beam.io.ReadFromText(options.input)
+ | 'ReadLines' >> beam.io.ReadFromText(args.input_file)
# Count the words.
| CountWords()
# Write the formatted word counts to output.
- | 'WriteCounts' >> beam.io.WriteToText(options.output))
+ | 'WriteCounts' >> beam.io.WriteToText(args.output_path))
# [END pipeline_monitoring_execution]
- p.visit(SnippetUtils.RenameFiles(renames))
-
-def examples_wordcount_minimal(renames):
+def examples_wordcount_minimal():
"""MinimalWordCount example snippets."""
import re
import apache_beam as beam
- from apache_beam.options.pipeline_options import GoogleCloudOptions
- from apache_beam.options.pipeline_options import StandardOptions
+ # [START examples_wordcount_minimal_options]
from apache_beam.options.pipeline_options import PipelineOptions
- # [START examples_wordcount_minimal_options]
- options = PipelineOptions()
- google_cloud_options = options.view_as(GoogleCloudOptions)
- google_cloud_options.project = 'my-project-id'
- google_cloud_options.job_name = 'myjob'
- google_cloud_options.staging_location = 'gs://your-bucket-name-here/staging'
- google_cloud_options.temp_location = 'gs://your-bucket-name-here/temp'
- options.view_as(StandardOptions).runner = 'DataflowRunner'
+ input_file = 'gs://dataflow-samples/shakespeare/kinglear.txt'
+ output_path = 'gs://my-bucket/counts.txt'
+
+ beam_options = PipelineOptions(
+ runner='DataflowRunner',
+ project='my-project-id',
+ job_name='unique-job-name',
+ temp_location='gs://my-bucket/temp',
+ )
# [END examples_wordcount_minimal_options]
# Run it locally for testing.
- options = PipelineOptions()
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--input-file')
+ parser.add_argument('--output-path')
+ args, beam_args = parser.parse_known_args()
+
+ input_file = args.input_file
+ output_path = args.output_path
+
+ beam_options = PipelineOptions(beam_args)
# [START examples_wordcount_minimal_create]
- p = beam.Pipeline(options=options)
+ pipeline = beam.Pipeline(options=beam_options)
# [END examples_wordcount_minimal_create]
(
# [START examples_wordcount_minimal_read]
- p
- | beam.io.ReadFromText('gs://dataflow-samples/shakespeare/kinglear.txt')
+ pipeline
+ | beam.io.ReadFromText(input_file)
# [END examples_wordcount_minimal_read]
# [START examples_wordcount_minimal_pardo]
@@ -456,54 +463,52 @@ def examples_wordcount_minimal(renames):
# [END examples_wordcount_minimal_map]
# [START examples_wordcount_minimal_write]
- | beam.io.WriteToText('gs://my-bucket/counts.txt')
+ | beam.io.WriteToText(output_path)
# [END examples_wordcount_minimal_write]
)
- p.visit(SnippetUtils.RenameFiles(renames))
-
# [START examples_wordcount_minimal_run]
- result = p.run()
+ result = pipeline.run()
# [END examples_wordcount_minimal_run]
result.wait_until_finish()
-def examples_wordcount_wordcount(renames):
+def examples_wordcount_wordcount():
"""WordCount example snippets."""
import re
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
- argv = []
-
# [START examples_wordcount_wordcount_options]
- class WordCountOptions(PipelineOptions):
- @classmethod
- def _add_argparse_args(cls, parser):
- parser.add_argument(
- '--input',
- help='Input for the pipeline',
- default='gs://my-bucket/input')
+ import argparse
- options = PipelineOptions(argv)
- word_count_options = options.view_as(WordCountOptions)
- with beam.Pipeline(options=options) as p:
- lines = p | beam.io.ReadFromText(word_count_options.input)
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--input-file',
+ default='gs://dataflow-samples/shakespeare/kinglear.txt',
+ help='The file path for the input text to process.')
+ parser.add_argument(
+ '--output-path', required=True, help='The path prefix for output files.')
+ args, beam_args = parser.parse_known_args()
+
+ beam_options = PipelineOptions(beam_args)
+ with beam.Pipeline(options=beam_options) as pipeline:
+ lines = pipeline | beam.io.ReadFromText(args.input_file)
# [END examples_wordcount_wordcount_options]
# [START examples_wordcount_wordcount_composite]
- class CountWords(beam.PTransform):
- def expand(self, pcoll):
- return (
- pcoll
- # Convert lines of text into individual words.
- | 'ExtractWords' >>
- beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
+ @beam.ptransform_fn
+ def CountWords(pcoll):
+ return (
+ pcoll
+ # Convert lines of text into individual words.
+ | 'ExtractWords' >>
+ beam.FlatMap(lambda x: re.findall(r'[A-Za-z\']+', x))
- # Count the number of times each word occurs.
- | beam.combiners.Count.PerElement())
+ # Count the number of times each word occurs.
+ | beam.combiners.Count.PerElement())
counts = lines | CountWords()
@@ -518,11 +523,10 @@ def examples_wordcount_wordcount(renames):
formatted = counts | beam.ParDo(FormatAsTextFn())
# [END examples_wordcount_wordcount_dofn]
- formatted | beam.io.WriteToText('gs://my-bucket/counts.txt')
- p.visit(SnippetUtils.RenameFiles(renames))
+ formatted | beam.io.WriteToText(args.output_path)
-def examples_wordcount_templated(renames):
+def examples_wordcount_templated():
"""Templated WordCount example snippet."""
import re
@@ -538,15 +542,19 @@ def examples_wordcount_templated(renames):
# Use add_value_provider_argument for arguments to be templatable
# Use add_argument as usual for non-templatable arguments
parser.add_value_provider_argument(
- '--input', help='Path of the file to read from')
+ '--input-file',
+ default='gs://dataflow-samples/shakespeare/kinglear.txt',
+ help='The file path for the input text to process.')
parser.add_argument(
- '--output', required=True, help='Output file to write results to.')
+ '--output-path',
+ required=True,
+ help='The path prefix for output files.')
- pipeline_options = PipelineOptions(['--output', 'some/output_path'])
- with beam.Pipeline(options=pipeline_options) as p:
+ beam_options = PipelineOptions()
+ args = beam_options.view_as(WordcountTemplatedOptions)
- wordcount_options = pipeline_options.view_as(WordcountTemplatedOptions)
- lines = p | 'Read' >> ReadFromText(wordcount_options.input)
+ with beam.Pipeline(options=beam_options) as pipeline:
+ lines = pipeline | 'Read' >> ReadFromText(args.input_file.get())
# [END example_wordcount_templated]
@@ -563,9 +571,7 @@ def examples_wordcount_templated(renames):
|
'Sum' >> beam.Map(lambda word_ones: (word_ones[0], sum(word_ones[1])))
| 'Format' >> beam.Map(format_result)
- | 'Write' >> WriteToText(wordcount_options.output))
-
- p.visit(SnippetUtils.RenameFiles(renames))
+ | 'Write' >> WriteToText(args.output_path))
def examples_wordcount_debugging(renames):
@@ -612,9 +618,9 @@ def examples_wordcount_debugging(renames):
# [END example_wordcount_debugging_logging]
# [END example_wordcount_debugging_aggregators]
- with TestPipeline() as p: # Use TestPipeline for testing.
+ with TestPipeline() as pipeline: # Use TestPipeline for testing.
filtered_words = (
- p
+ pipeline
|
beam.io.ReadFromText('gs://dataflow-samples/shakespeare/kinglear.txt')
|
@@ -638,14 +644,13 @@ def examples_wordcount_debugging(renames):
| 'format' >> beam.Map(format_result)
| 'Write' >> beam.io.WriteToText('gs://my-bucket/counts.txt'))
- p.visit(SnippetUtils.RenameFiles(renames))
+ pipeline.visit(SnippetUtils.RenameFiles(renames))
-def examples_wordcount_streaming(argv):
+def examples_wordcount_streaming():
import apache_beam as beam
from apache_beam import window
from apache_beam.options.pipeline_options import PipelineOptions
- from apache_beam.options.pipeline_options import StandardOptions
# Parse out arguments.
parser = argparse.ArgumentParser()
@@ -666,19 +671,18 @@ def examples_wordcount_streaming(argv):
help=(
'Input PubSub subscription of the form '
'"projects/<PROJECT>/subscriptions/<SUBSCRIPTION>."'))
- known_args, pipeline_args = parser.parse_known_args(argv)
+ args, beam_args = parser.parse_known_args()
- pipeline_options = PipelineOptions(pipeline_args)
- pipeline_options.view_as(StandardOptions).streaming = True
+ beam_options = PipelineOptions(beam_args, streaming=True)
- with TestPipeline(options=pipeline_options) as p:
+ with TestPipeline(options=beam_options) as pipeline:
# [START example_wordcount_streaming_read]
# Read from Pub/Sub into a PCollection.
- if known_args.input_subscription:
- lines = p | beam.io.ReadFromPubSub(
- subscription=known_args.input_subscription)
+ if args.input_subscription:
+ lines = pipeline | beam.io.ReadFromPubSub(
+ subscription=args.input_subscription)
else:
- lines = p | beam.io.ReadFromPubSub(topic=known_args.input_topic)
+ lines = pipeline | beam.io.ReadFromPubSub(topic=args.input_topic)
# [END example_wordcount_streaming_read]
output = (
@@ -696,7 +700,7 @@ def examples_wordcount_streaming(argv):
# [START example_wordcount_streaming_write]
# Write to Pub/Sub
- output | beam.io.WriteToPubSub(known_args.output_topic)
+ output | beam.io.WriteToPubSub(args.output_topic)
# [END example_wordcount_streaming_write]
@@ -719,13 +723,13 @@ def examples_ptransforms_templated(renames):
def process(self, an_int):
yield self.templated_int.get() + an_int
- pipeline_options = PipelineOptions()
- with beam.Pipeline(options=pipeline_options) as p:
+ beam_options = PipelineOptions()
+ args = beam_options.view_as(TemplatedUserOptions)
- user_options = pipeline_options.view_as(TemplatedUserOptions)
- my_sum_fn = MySumFn(user_options.templated_int)
+ with beam.Pipeline(options=beam_options) as pipeline:
+ my_sum_fn = MySumFn(args.templated_int)
sum = (
- p
+ pipeline
| 'ReadCollection' >>
beam.io.ReadFromText('gs://some/integer_collection')
| 'StringToInt' >> beam.Map(lambda w: int(w))
@@ -737,7 +741,7 @@ def examples_ptransforms_templated(renames):
# so a value must be provided at graph-construction time
my_sum_fn.templated_int = StaticValueProvider(int, 10)
- p.visit(SnippetUtils.RenameFiles(renames))
+ pipeline.visit(SnippetUtils.RenameFiles(renames))
# Defining a new source.
@@ -834,8 +838,8 @@ def model_custom_source(count):
# Using the source in an example pipeline.
# [START model_custom_source_use_new_source]
- with beam.Pipeline(options=PipelineOptions()) as p:
- numbers = p | 'ProduceNumbers' >> beam.io.Read(CountingSource(count))
+ with beam.Pipeline() as pipeline:
+ numbers = pipeline | 'ProduceNumbers' >> beam.io.Read(CountingSource(count))
# [END model_custom_source_use_new_source]
lines = numbers | beam.core.Map(lambda number: 'line %d' % number)
@@ -843,8 +847,8 @@ def model_custom_source(count):
lines, equal_to(['line ' + str(number) for number in range(0, count)]))
# [START model_custom_source_use_ptransform]
- with beam.Pipeline(options=PipelineOptions()) as p:
- numbers = p | 'ProduceNumbers' >> ReadFromCountingSource(count)
+ with beam.Pipeline() as pipeline:
+ numbers = pipeline | 'ProduceNumbers' >> ReadFromCountingSource(count)
# [END model_custom_source_use_ptransform]
lines = numbers | beam.core.Map(lambda number: 'line %d' % number)
@@ -978,8 +982,8 @@ def model_custom_sink(
# Using the new sink in an example pipeline.
# [START model_custom_sink_use_new_sink]
- with beam.Pipeline(options=PipelineOptions()) as p:
- kvs = p | 'CreateKVs' >> beam.Create(KVs)
+ with beam.Pipeline(options=PipelineOptions()) as pipeline:
+ kvs = pipeline | 'CreateKVs' >> beam.Create(KVs)
kvs | 'WriteToSimpleKV' >> beam.io.Write(
SimpleKVSink(simplekv, 'http://url_to_simple_kv/', final_table_name))
@@ -988,8 +992,8 @@ def model_custom_sink(
final_table_name = final_table_name_with_ptransform
# [START model_custom_sink_use_ptransform]
- with beam.Pipeline(options=PipelineOptions()) as p:
- kvs = p | 'CreateKVs' >> beam.core.Create(KVs)
+ with beam.Pipeline(options=PipelineOptions()) as pipeline:
+ kvs = pipeline | 'CreateKVs' >> beam.core.Create(KVs)
kvs | 'WriteToSimpleKV' >> WriteToKVSink(
simplekv, 'http://url_to_simple_kv/', final_table_name)
# [END model_custom_sink_use_ptransform]
@@ -1002,9 +1006,10 @@ def model_textio(renames):
return re.findall(r'[A-Za-z\']+', x)
# [START model_textio_read]
- with beam.Pipeline(options=PipelineOptions()) as p:
+ with beam.Pipeline(options=PipelineOptions()) as pipeline:
# [START model_pipelineio_read]
- lines = p | 'ReadFromText' >> beam.io.ReadFromText('path/to/input-*.csv')
+ lines = pipeline | 'ReadFromText' >> beam.io.ReadFromText(
+ 'path/to/input-*.csv')
# [END model_pipelineio_read]
# [END model_textio_read]
@@ -1016,21 +1021,21 @@ def model_textio(renames):
# [END model_pipelineio_write]
# [END model_textio_write]
- p.visit(SnippetUtils.RenameFiles(renames))
+ pipeline.visit(SnippetUtils.RenameFiles(renames))
def model_textio_compressed(renames, expected):
"""Using a Read Transform to read compressed text files."""
- with TestPipeline() as p:
+ with TestPipeline() as pipeline:
# [START model_textio_write_compressed]
- lines = p | 'ReadFromText' >> beam.io.ReadFromText(
+ lines = pipeline | 'ReadFromText' >> beam.io.ReadFromText(
'/path/to/input-*.csv.gz',
compression_type=beam.io.filesystem.CompressionTypes.GZIP)
# [END model_textio_write_compressed]
assert_that(lines, equal_to(expected))
- p.visit(SnippetUtils.RenameFiles(renames))
+ pipeline.visit(SnippetUtils.RenameFiles(renames))
def model_datastoreio():
@@ -1050,13 +1055,13 @@ def model_datastoreio():
query = Query(kind, project)
# [START model_datastoreio_read]
- p = beam.Pipeline(options=PipelineOptions())
- entities = p | 'Read From Datastore' >> ReadFromDatastore(query)
+ pipeline = beam.Pipeline(options=PipelineOptions())
+ entities = pipeline | 'Read From Datastore' >> ReadFromDatastore(query)
# [END model_datastoreio_read]
# [START model_datastoreio_write]
- p = beam.Pipeline(options=PipelineOptions())
- musicians = p | 'Musicians' >> beam.Create(
+ pipeline = beam.Pipeline(options=PipelineOptions())
+ musicians = pipeline | 'Musicians' >> beam.Create(
['Mozart', 'Chopin', 'Beethoven', 'Vivaldi'])
def to_entity(content):
@@ -1070,7 +1075,8 @@ def model_datastoreio():
# [END model_datastoreio_write]
-def model_bigqueryio(p, write_project='', write_dataset='', write_table=''):
+def model_bigqueryio(
+ pipeline, write_project='', write_dataset='', write_table=''):
"""Using a Read and Write transform to read/write from/to BigQuery."""
# [START model_bigqueryio_table_spec]
@@ -1110,7 +1116,7 @@ def model_bigqueryio(p, write_project='', write_dataset='', write_table=''):
# [START model_bigqueryio_read_table]
max_temperatures = (
- p
+ pipeline
| 'ReadTable' >> beam.io.ReadFromBigQuery(table=table_spec)
# Each row is a dictionary where the keys are the BigQuery columns
| beam.Map(lambda elem: elem['max_temperature']))
@@ -1118,7 +1124,7 @@ def model_bigqueryio(p, write_project='', write_dataset='', write_table=''):
# [START model_bigqueryio_read_query]
max_temperatures = (
- p
+ pipeline
| 'QueryTable' >> beam.io.ReadFromBigQuery(
query='SELECT max_temperature FROM '\
'[clouddataflow-readonly:samples.weather_stations]')
@@ -1128,7 +1134,7 @@ def model_bigqueryio(p, write_project='', write_dataset='', write_table=''):
# [START model_bigqueryio_read_query_std_sql]
max_temperatures = (
- p
+ pipeline
| 'QueryTableStdSQL' >> beam.io.ReadFromBigQuery(
query='SELECT max_temperature FROM '\
'`clouddataflow-readonly.samples.weather_stations`',
@@ -1156,7 +1162,7 @@ def model_bigqueryio(p, write_project='', write_dataset='', write_table=''):
table_spec = '{}:{}.{}'.format(write_project, write_dataset, write_table)
# [START model_bigqueryio_write_input]
- quotes = p | beam.Create([
+ quotes = pipeline | beam.Create([
{
'source': 'Mahatma Gandhi', 'quote': 'My life is my message.'
},
@@ -1176,8 +1182,8 @@ def model_bigqueryio(p, write_project='', write_dataset='', write_table=''):
# [START model_bigqueryio_write_dynamic_destinations]
fictional_characters_view = beam.pvalue.AsDict(
- p | 'CreateCharacters' >> beam.Create([('Yoda', True),
- ('Obi Wan Kenobi', True)]))
+ pipeline | 'CreateCharacters' >> beam.Create([('Yoda', True),
+ ('Obi Wan Kenobi', True)]))
def table_fn(element, fictional_characters):
if element in fictional_characters:
@@ -1233,9 +1239,9 @@ def model_composite_transform_example(contents, output_path):
# [END composite_ptransform_apply_method]
# [END composite_transform_example]
- with TestPipeline() as p: # Use TestPipeline for testing.
+ with TestPipeline() as pipeline: # Use TestPipeline for testing.
(
- p
+ pipeline
| beam.Create(contents)
| CountWords()
| beam.io.WriteToText(output_path))
@@ -1246,10 +1252,11 @@ def model_multiple_pcollections_flatten(contents, output_path):
some_hash_fn = lambda s: ord(s[0])
partition_fn = lambda element, partitions: some_hash_fn(element) % partitions
import apache_beam as beam
- with TestPipeline() as p: # Use TestPipeline for testing.
+ with TestPipeline() as pipeline: # Use TestPipeline for testing.
# Partition into deciles
- partitioned = p | beam.Create(contents) | beam.Partition(partition_fn, 3)
+ partitioned = pipeline | beam.Create(contents) | beam.Partition(
+ partition_fn, 3)
pcoll1 = partitioned[0]
pcoll2 = partitioned[1]
pcoll3 = partitioned[2]
@@ -1279,9 +1286,9 @@ def model_multiple_pcollections_partition(contents, output_path):
return i
import apache_beam as beam
- with TestPipeline() as p: # Use TestPipeline for testing.
+ with TestPipeline() as pipeline: # Use TestPipeline for testing.
- students = p | beam.Create(contents)
+ students = pipeline | beam.Create(contents)
# [START model_multiple_pcollections_partition]
def partition_fn(student, num_partitions):
@@ -1303,14 +1310,14 @@ def model_group_by_key(contents, output_path):
import re
import apache_beam as beam
- with TestPipeline() as p: # Use TestPipeline for testing.
+ with TestPipeline() as pipeline: # Use TestPipeline for testing.
def count_ones(word_ones):
(word, ones) = word_ones
return (word, sum(ones))
words_and_counts = (
- p
+ pipeline
| beam.Create(contents)
| beam.FlatMap(lambda x: re.findall(r'\w+', x))
| 'one word' >> beam.Map(lambda w: (w, 1)))
@@ -1356,14 +1363,14 @@ def model_join_using_side_inputs(
import apache_beam as beam
from apache_beam.pvalue import AsIter
- with TestPipeline() as p: # Use TestPipeline for testing.
+ with TestPipeline() as pipeline: # Use TestPipeline for testing.
# [START model_join_using_side_inputs]
# This code performs a join by receiving the set of names as an input and
# passing PCollections that contain emails and phone numbers as side inputs
# instead of using CoGroupByKey.
- names = p | 'names' >> beam.Create(name_list)
- emails = p | 'email' >> beam.Create(email_list)
- phones = p | 'phone' >> beam.Create(phone_list)
+ names = pipeline | 'names' >> beam.Create(name_list)
+ emails = pipeline | 'email' >> beam.Create(email_list)
+ phones = pipeline | 'phone' >> beam.Create(phone_list)
def join_info(name, emails, phone_numbers):
filtered_emails = []
@@ -1416,9 +1423,9 @@ def file_process_pattern_access_metadata():
from apache_beam.io import fileio
# [START FileProcessPatternAccessMetadataSnip1]
- with beam.Pipeline() as p:
+ with beam.Pipeline() as pipeline:
readable_files = (
- p
+ pipeline
| fileio.MatchFiles('hdfs://path/to/*.txt')
| fileio.ReadMatches()
| beam.Reshuffle())
@@ -1456,21 +1463,21 @@ def accessing_valueprovider_info_after_run():
'The string value is %s' %
RuntimeValueProvider.get_value('string_value', str, ''))
- pipeline_options = PipelineOptions()
+ beam_options = PipelineOptions()
+ args = beam_options.view_as(MyOptions)
+
# Create pipeline.
- with beam.Pipeline(options=pipeline_options) as p:
+ with beam.Pipeline(options=beam_options) as pipeline:
- my_options = pipeline_options.view_as(MyOptions)
# Add a branch for logging the ValueProvider value.
_ = (
- p
+ pipeline
| beam.Create([None])
- | 'LogValueProvs' >> beam.ParDo(
- LogValueProvidersFn(my_options.string_value)))
+ | 'LogValueProvs' >> beam.ParDo(LogValueProvidersFn(args.string_value)))
# The main pipeline.
result_pc = (
- p
+ pipeline
| "main_pc" >> beam.Create([1, 2, 3])
| beam.combiners.Sum.Globally())
@@ -1499,17 +1506,16 @@ def side_input_slow_update(
yield (left, x)
# Create pipeline.
- pipeline_options = PipelineOptions()
- p = beam.Pipeline(options=pipeline_options)
+ pipeline = beam.Pipeline()
side_input = (
- p
+ pipeline
| 'PeriodicImpulse' >> PeriodicImpulse(
first_timestamp, last_timestamp, interval, True)
| 'MapToFileName' >> beam.Map(lambda x: src_file_pattern + str(x))
| 'ReadFromFile' >> beam.io.ReadAllFromText())
main_input = (
- p
+ pipeline
| 'MpImpulse' >> beam.Create(sample_main_input_elements)
|
'MapMpToTimestamped' >> beam.Map(lambda src: TimestampedValue(src, src))
@@ -1522,7 +1528,7 @@ def side_input_slow_update(
cross_join, rights=beam.pvalue.AsIter(side_input)))
# [END SideInputSlowUpdateSnip1]
- return p, result
+ return pipeline, result
def bigqueryio_deadletter():
@@ -1531,10 +1537,10 @@ def bigqueryio_deadletter():
# Create pipeline.
schema = ({'fields': [{'name': 'a', 'type': 'STRING', 'mode': 'REQUIRED'}]})
- p = beam.Pipeline()
+ pipeline = beam.Pipeline()
errors = (
- p | 'Data' >> beam.Create([1, 2])
+ pipeline | 'Data' >> beam.Create([1, 2])
| 'CreateBrokenData' >>
beam.Map(lambda src: {'a': src} if src == 2 else {'a': None})
| 'WriteToBigQuery' >> beam.io.WriteToBigQuery(
@@ -1605,9 +1611,9 @@ def nlp_analyze_text():
extract_syntax=True,
)
- with beam.Pipeline() as p:
+ with beam.Pipeline() as pipeline:
responses = (
- p
+ pipeline
| beam.Create([
'My experience so far has been fantastic! '
'I\'d really recommend this product.'
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index fddb24c..5c7bd46 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -25,6 +25,7 @@ import gzip
import logging
import math
import os
+import sys
import tempfile
import time
import unittest
@@ -600,15 +601,20 @@ class SnippetsTest(unittest.TestCase):
def test_model_pipelines(self):
temp_path = self.create_temp_file('aa bb cc\n bb cc\n cc')
result_path = temp_path + '.result'
- snippets.model_pipelines(
- ['--input=%s*' % temp_path, '--output=%s' % result_path])
+ test_argv = [
+ "unused_argv[0]",
+ f"--input-file={temp_path}*",
+ f"--output-path={result_path}",
+ ]
+ with mock.patch.object(sys, 'argv', test_argv):
+ snippets.model_pipelines()
self.assertEqual(
self.get_output(result_path),
[str(s) for s in [(u'aa', 1), (u'bb', 2), (u'cc', 3)]])
def test_model_pcollection(self):
temp_path = self.create_temp_file()
- snippets.model_pcollection(['--output=%s' % temp_path])
+ snippets.model_pcollection(temp_path)
self.assertEqual(
self.get_output(temp_path),
[
@@ -751,7 +757,13 @@ class SnippetsTest(unittest.TestCase):
def _run_test_pipeline_for_options(self, fn):
temp_path = self.create_temp_file('aa\nbb\ncc')
result_path = temp_path + '.result'
- fn(['--input=%s*' % temp_path, '--output=%s' % result_path])
+ test_argv = [
+ "unused_argv[0]",
+ f"--input-file={temp_path}*",
+ f"--output-path={result_path}",
+ ]
+ with mock.patch.object(sys, 'argv', test_argv):
+ fn()
self.assertEqual(['aa', 'bb', 'cc'], self.get_output(result_path))
def test_pipeline_options_local(self):
@@ -783,7 +795,13 @@ class SnippetsTest(unittest.TestCase):
def test_examples_wordcount(self, pipeline):
temp_path = self.create_temp_file('abc def ghi\n abc jkl')
result_path = self.create_temp_file()
- pipeline({'read': temp_path, 'write': result_path})
+ test_argv = [
+ "unused_argv[0]",
+ f"--input-file={temp_path}*",
+ f"--output-path={result_path}",
+ ]
+ with mock.patch.object(sys, 'argv', test_argv):
+ pipeline()
self.assertEqual(
self.get_output(result_path), ['abc: 2', 'def: 1', 'ghi: 1', 'jkl: 1'])
@@ -850,23 +868,29 @@ class SnippetsTest(unittest.TestCase):
FakeReadFromPubSub(topic=input_topic, values=input_values))
beam.io.WriteToPubSub = (
FakeWriteToPubSub(topic=output_topic, values=output_values))
- snippets.examples_wordcount_streaming([
+ test_argv = [
+ 'unused_argv[0]',
'--input_topic',
'projects/fake-beam-test-project/topic/intopic',
'--output_topic',
'projects/fake-beam-test-project/topic/outtopic'
- ])
+ ]
+ with mock.patch.object(sys, 'argv', test_argv):
+ snippets.examples_wordcount_streaming()
# Test with custom subscription.
input_sub = 'projects/fake-beam-test-project/subscriptions/insub'
beam.io.ReadFromPubSub = FakeReadFromPubSub(
subscription=input_sub, values=input_values)
- snippets.examples_wordcount_streaming([
+ test_argv = [
+ 'unused_argv[0]',
'--input_subscription',
'projects/fake-beam-test-project/subscriptions/insub',
'--output_topic',
'projects/fake-beam-test-project/topic/outtopic'
- ])
+ ]
+ with mock.patch.object(sys, 'argv', test_argv):
+ snippets.examples_wordcount_streaming()
def test_model_composite_transform_example(self):
contents = ['aa bb cc', 'bb cc', 'cc']