You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/02/06 11:44:11 UTC
[beam] branch asf-site updated: Publishing website 2020/02/06
11:44:03 at commit b91560c
This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 972368f Publishing website 2020/02/06 11:44:03 at commit b91560c
972368f is described below
commit 972368fbbf0d9bf77e89e216223ea4ef4e4b2e82
Author: jenkins <bu...@apache.org>
AuthorDate: Thu Feb 6 11:44:03 2020 +0000
Publishing website 2020/02/06 11:44:03 at commit b91560c
---
.../io/built-in/google-bigquery/index.html | 19 +-
.../io/developing-io-python/index.html | 23 ++-
.../patterns/file-processing/index.html | 15 +-
.../patterns/pipeline-options/index.html | 22 +-
.../documentation/programming-guide/index.html | 124 +++++++-----
.../sdks/python-type-safety/index.html | 4 +-
.../python/elementwise/filter/index.html | 151 +++++++++-----
.../python/elementwise/flatmap/index.html | 70 ++++---
.../transforms/python/elementwise/keys/index.html | 3 +-
.../python/elementwise/kvswap/index.html | 3 +-
.../transforms/python/elementwise/map/index.html | 54 ++---
.../transforms/python/elementwise/pardo/index.html | 31 +--
.../python/elementwise/partition/index.html | 14 +-
.../transforms/python/elementwise/regex/index.html | 27 +--
.../python/elementwise/tostring/index.html | 9 +-
.../python/elementwise/values/index.html | 3 +-
.../get-started/mobile-gaming-example/index.html | 225 ++++++++++++---------
.../get-started/wordcount-example/index.html | 31 +--
18 files changed, 475 insertions(+), 353 deletions(-)
diff --git a/website/generated-content/documentation/io/built-in/google-bigquery/index.html b/website/generated-content/documentation/io/built-in/google-bigquery/index.html
index a0824cc..857222d 100644
--- a/website/generated-content/documentation/io/built-in/google-bigquery/index.html
+++ b/website/generated-content/documentation/io/built-in/google-bigquery/index.html
@@ -1097,10 +1097,13 @@ two fields (source and quote) of type string.</p>
<span class="o">.</span><span class="na">setType</span><span class="o">(</span><span class="s">"STRING"</span><span class="o">)</span>
<span class="o">.</span><span class="na">setMode</span><span class="o">(</span><span class="s">"REQUIRED"</span><span class="o">)));</span>
</code></pre></div></div>
-<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">table_schema</span> <span class="o">=</span> <span class="p">{</span><span class="s">'fields'</span><span class="p">:</span> <span class="p">[</span>
- <span class="p">{</span><span class="s">'name'</span><span class="p">:</span> <span class="s">'source'</span><span class="p">,</span> <span class="s">'type'</span><span class="p">:</span> <span class="s">'STRING'</span><span class="p">,</span> <span class="s">'mode'</span><span class="p">:</span> <span class="s">'NULLABLE'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'name'</span><span class="p">:</span> <span class="s">'quote'</span><span class="p">,</span> <span class="s">'type'</span><span class="p">:</span> <span class="s">'STRING'</span><span class="p">,</span> <span class="s">'mode'</span><span class="p">:</span> <span class="s">'REQUIRED'</span><span class="p">}</span>
-<span class="p">]}</span>
+<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">table_schema</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s">'fields'</span><span class="p">:</span> <span class="p">[{</span>
+ <span class="s">'name'</span><span class="p">:</span> <span class="s">'source'</span><span class="p">,</span> <span class="s">'type'</span><span class="p">:</span> <span class="s">'STRING'</span><span class="p">,</span> <span class="s">'mode'</span><span class="p">:</span> <span class="s">'NULLABLE'</span>
+ <span class="p">},</span> <span class="p">{</span>
+ <span class="s">'name'</span><span class="p">:</span> <span class="s">'quote'</span><span class="p">,</span> <span class="s">'type'</span><span class="p">:</span> <span class="s">'STRING'</span><span class="p">,</span> <span class="s">'mode'</span><span class="p">:</span> <span class="s">'REQUIRED'</span>
+ <span class="p">}]</span>
+<span class="p">}</span>
</code></pre></div></div>
<h4 id="using-a-string-1">Using a string</h4>
@@ -1250,8 +1253,12 @@ static class Quote {
<span class="k">new</span> <span class="nf">Quote</span><span class="o">(</span><span class="s">"Yoda"</span><span class="o">,</span> <span class="s">"Do, or do not. There is no 'try'."</span><span class="o">)));</span>
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">quotes</span> <span class="o">=</span> <span class="n">p</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'source'</span><span class="p">:</span> <span class="s">'Mahatma Gandhi'</span><span class="p">,</span> <span class="s">'quote'</span><span class="p">:</span> <span class="s">'My life is my message.'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'source'</span><span class="p">:</span> <span class="s">'Yoda'</span><span class="p">,</span> <span class="s">'quote'</span><span class="p">:</span> <span class="s">"Do, or do not. There is no 'try'."</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'source'</span><span class="p">:</span> <span class="s">'Mahatma Gandhi'</span><span class="p">,</span> <span class="s">'quote'</span><span class="p">:</span> <span class="s">'My life is my message.'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'source'</span><span class="p">:</span> <span class="s">'Yoda'</span><span class="p">,</span> <span class="s">'quote'</span><span class="p">:</span> <span class="s">"Do, or do not. There is no 'try'."</span>
+ <span class="p">},</span>
<span class="p">])</span>
</code></pre></div></div>
diff --git a/website/generated-content/documentation/io/developing-io-python/index.html b/website/generated-content/documentation/io/developing-io-python/index.html
index ba8b41c..05006ef 100644
--- a/website/generated-content/documentation/io/developing-io-python/index.html
+++ b/website/generated-content/documentation/io/developing-io-python/index.html
@@ -693,7 +693,6 @@ a wrapper.</li>
<p>The following example, <code class="highlighter-rouge">CountingSource</code>, demonstrates an implementation of <code class="highlighter-rouge">BoundedSource</code> and uses the SDK-provided <code class="highlighter-rouge">RangeTracker</code> called <code class="highlighter-rouge">OffsetRangeTracker</code>.</p>
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>class CountingSource(iobase.BoundedSource):
-
def __init__(self, count):
self.records_read = Metrics.counter(self.__class__, 'recordsRead')
self._count = count
@@ -726,11 +725,14 @@ a wrapper.</li>
bundle_start = start_position
while bundle_start < stop_position:
bundle_stop = min(stop_position, bundle_start + desired_bundle_size)
- yield iobase.SourceBundle(weight=(bundle_stop - bundle_start),
- source=self,
- start_position=bundle_start,
- stop_position=bundle_stop)
+ yield iobase.SourceBundle(
+ weight=(bundle_stop - bundle_start),
+ source=self,
+ start_position=bundle_start,
+ stop_position=bundle_stop)
bundle_start = bundle_stop
+
+
</code></pre></div></div>
<p>To read data from the source in your pipeline, use the <code class="highlighter-rouge">Read</code> transform:</p>
@@ -816,13 +818,14 @@ to <code class="highlighter-rouge">_CountingSource</code>. Then, create the wrap
<code class="highlighter-rouge">ReadFromCountingSource</code>:</p>
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>class ReadFromCountingSource(PTransform):
-
def __init__(self, count):
super(ReadFromCountingSource, self).__init__()
self._count = count
def expand(self, pcoll):
return pcoll | iobase.Read(_CountingSource(self._count))
+
+
</code></pre></div></div>
<p>Finally, read from the source:</p>
@@ -834,7 +837,6 @@ to <code class="highlighter-rouge">_CountingSource</code>. Then, create the wrap
<p>For the sink, rename <code class="highlighter-rouge">SimpleKVSink</code> to <code class="highlighter-rouge">_SimpleKVSink</code>. Then, create the wrapper <code class="highlighter-rouge">PTransform</code>, called <code class="highlighter-rouge">WriteToKVSink</code>:</p>
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>class WriteToKVSink(PTransform):
-
def __init__(self, simplekv, url, final_table_name):
self._simplekv = simplekv
super(WriteToKVSink, self).__init__()
@@ -842,9 +844,10 @@ to <code class="highlighter-rouge">_CountingSource</code>. Then, create the wrap
self._final_table_name = final_table_name
def expand(self, pcoll):
- return pcoll | iobase.Write(_SimpleKVSink(self._simplekv,
- self._url,
- self._final_table_name))
+ return pcoll | iobase.Write(
+ _SimpleKVSink(self._simplekv, self._url, self._final_table_name))
+
+
</code></pre></div></div>
<p>Finally, write to the sink:</p>
diff --git a/website/generated-content/documentation/patterns/file-processing/index.html b/website/generated-content/documentation/patterns/file-processing/index.html
index 27a0569..52bb13e 100644
--- a/website/generated-content/documentation/patterns/file-processing/index.html
+++ b/website/generated-content/documentation/patterns/file-processing/index.html
@@ -593,13 +593,14 @@ limitations under the License.
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">with</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">()</span> <span class="k">as</span> <span class="n">p</span><span class="p">:</span>
- <span class="n">readable_files</span> <span class="o">=</span> <span class="p">(</span><span class="n">p</span>
- <span class="o">|</span> <span class="n">fileio</span><span class="o">.</span><span class="n">MatchFiles</span><span class="p">(</span><span class="s">'hdfs://path/to/*.txt'</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">fileio</span><span class="o">.</span><span class="n">ReadMatches</span><span class="p">()</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Reshuffle</span><span class="p">())</span>
- <span class="n">files_and_contents</span> <span class="o">=</span> <span class="p">(</span><span class="n">readable_files</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">path</span><span class="p">,</span>
- <span class="n">x</span><span class="o">.</span><span class="n">read_utf8</span><span class="p">())))</span>
+ <span class="n">readable_files</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">p</span>
+ <span class="o">|</span> <span class="n">fileio</span><span class="o">.</span><span class="n">MatchFiles</span><span class="p">(</span><span class="s">'hdfs://path/to/*.txt'</span><span class="p">)</span>
+ <span class="o">|</span> <span class="n">fileio</span><span class="o">.</span><span class="n">ReadMatches</span><span class="p">()</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Reshuffle</span><span class="p">())</span>
+ <span class="n">files_and_contents</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">readable_files</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">read_utf8</span><span class="p">())))</span>
</code></pre></div></div>
diff --git a/website/generated-content/documentation/patterns/pipeline-options/index.html b/website/generated-content/documentation/patterns/pipeline-options/index.html
index 57b3186..3814925 100644
--- a/website/generated-content/documentation/patterns/pipeline-options/index.html
+++ b/website/generated-content/documentation/patterns/pipeline-options/index.html
@@ -566,8 +566,9 @@ limitations under the License.
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">an_int</span><span class="p">):</span>
<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">'The string_value is </span><span class="si">%</span><span class="s">s'</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">string_vp</span><span class="o">.</span><span class="n">get</span><span class="p">())</span>
<span class="c"># Another option (where you don't need to pass the value at all) is:</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">'The string value is </span><span class="si">%</span><span class="s">s'</span> <span class="o">%</span>
- <span class="n">RuntimeValueProvider</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s">'string_value'</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="s">''</span><span class="p">))</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s">'The string value is </span><span class="si">%</span><span class="s">s'</span> <span class="o">%</span>
+ <span class="n">RuntimeValueProvider</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s">'string_value'</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="s">''</span><span class="p">))</span>
<span class="n">pipeline_options</span> <span class="o">=</span> <span class="n">PipelineOptions</span><span class="p">()</span>
<span class="c"># Create pipeline.</span>
@@ -575,16 +576,17 @@ limitations under the License.
<span class="n">my_options</span> <span class="o">=</span> <span class="n">pipeline_options</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">MyOptions</span><span class="p">)</span>
<span class="c"># Add a branch for logging the ValueProvider value.</span>
- <span class="n">_</span> <span class="o">=</span> <span class="p">(</span><span class="n">p</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="bp">None</span><span class="p">])</span>
- <span class="o">|</span> <span class="s">'LogValueProvs'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span>
- <span class="n">LogValueProvidersFn</span><span class="p">(</span><span class="n">my_options</span><span class="o">.</span><span class="n">string_value</span><span class="p">)))</span>
+ <span class="n">_</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">p</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="bp">None</span><span class="p">])</span>
+ <span class="o">|</span> <span class="s">'LogValueProvs'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span>
+ <span class="n">LogValueProvidersFn</span><span class="p">(</span><span class="n">my_options</span><span class="o">.</span><span class="n">string_value</span><span class="p">)))</span>
<span class="c"># The main pipeline.</span>
- <span class="n">result_pc</span> <span class="o">=</span> <span class="p">(</span><span class="n">p</span>
- <span class="o">|</span> <span class="s">"main_pc"</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">Sum</span><span class="o">.</span><span class="n">Globally</span><span class="p">())</span>
-
+ <span class="n">result_pc</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">p</span>
+ <span class="o">|</span> <span class="s">"main_pc"</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">Sum</span><span class="o">.</span><span class="n">Globally</span><span class="p">())</span>
</code></pre></div></div>
diff --git a/website/generated-content/documentation/programming-guide/index.html b/website/generated-content/documentation/programming-guide/index.html
index 7d1dfae..95bdddf 100644
--- a/website/generated-content/documentation/programming-guide/index.html
+++ b/website/generated-content/documentation/programming-guide/index.html
@@ -738,12 +738,12 @@ adding <code class="highlighter-rouge">input</code> and <code class="highlighter
<span class="o">}</span>
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">class</span> <span class="nc">MyOptions</span><span class="p">(</span><span class="n">PipelineOptions</span><span class="p">):</span>
-
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">_add_argparse_args</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">parser</span><span class="p">):</span>
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--input'</span><span class="p">)</span>
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--output'</span><span class="p">)</span>
+
</code></pre></div></div>
<div class="language-go highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">var</span><span class="x"> </span><span class="p">(</span><span class="x">
</span><span class="n">input</span><span class="x"> </span><span class="o">=</span><span class="x"> </span><span class="n">flag</span><span class="o">.</span><span class="n">String</span><span class="p">(</span><span class="s">"input"</span><span class="p">,</span><span class="x"> </span><span class="s">""</span><span class="p">,</span><span class="x"> </span><span class="s">""</span><span class="p">)</span><span class="x">
@@ -769,15 +769,17 @@ a command-line argument, and a default value.</p>
<span class="o">}</span>
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">class</span> <span class="nc">MyOptions</span><span class="p">(</span><span class="n">PipelineOptions</span><span class="p">):</span>
-
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">_add_argparse_args</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">parser</span><span class="p">):</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--input'</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'Input for the pipeline'</span><span class="p">,</span>
- <span class="n">default</span><span class="o">=</span><span class="s">'gs://my-bucket/input'</span><span class="p">)</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--output'</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'Output for the pipeline'</span><span class="p">,</span>
- <span class="n">default</span><span class="o">=</span><span class="s">'gs://my-bucket/output'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--input'</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'Input for the pipeline'</span><span class="p">,</span>
+ <span class="n">default</span><span class="o">=</span><span class="s">'gs://my-bucket/input'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--output'</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'Output for the pipeline'</span><span class="p">,</span>
+ <span class="n">default</span><span class="o">=</span><span class="s">'gs://my-bucket/output'</span><span class="p">)</span>
+
</code></pre></div></div>
<div class="language-go highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">var</span><span class="x"> </span><span class="p">(</span><span class="x">
@@ -855,7 +857,8 @@ a <code class="highlighter-rouge">PCollection</code>:</p>
<span class="s">"ReadMyFile"</span><span class="o">,</span> <span class="n">TextIO</span><span class="o">.</span><span class="na">read</span><span class="o">().</span><span class="na">from</span><span class="o">(</span><span class="s">"gs://some/inputData.txt"</span><span class="o">));</span>
<span class="o">}</span>
</code></pre></div></div>
-<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">lines</span> <span class="o">=</span> <span class="n">p</span> <span class="o">|</span> <span class="s">'ReadMyFile'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">(</span><span class="s">'gs://some/inputData.txt'</span><span class="p">)</span>
+<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">lines</span> <span class="o">=</span> <span class="n">p</span> <span class="o">|</span> <span class="s">'ReadMyFile'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">(</span>
+ <span class="s">'gs://some/inputData.txt'</span><span class="p">)</span>
</code></pre></div></div>
<div class="language-go highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">lines</span><span class="x"> </span><span class="o">:=</span><span class="x"> </span><span class="n">textio</span><span class="o">.</span><span class="n">Read</span><span class="p">(</span><span class="n">s</span><span class="p">,</span><span class="x"> </span><span class="s">"gs://some/inputData.txt"</span><span class="p">)</span><span class="x">
@@ -911,8 +914,8 @@ itself.</p>
<span class="s">"Whether 'tis nobler in the mind to suffer "</span><span class="p">,</span>
<span class="s">'The slings and arrows of outrageous fortune, '</span><span class="p">,</span>
<span class="s">'Or to take arms against a sea of troubles, '</span><span class="p">,</span>
- <span class="p">])</span>
- <span class="p">)</span>
+ <span class="p">]))</span>
+
</code></pre></div></div>
@@ -1168,6 +1171,7 @@ following example code:</p>
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">element</span><span class="p">):</span>
<span class="k">return</span> <span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">element</span><span class="p">)]</span>
+
<span class="c"># Apply a ParDo to the PCollection "words" to compute lengths for each word.</span>
<span class="n">word_lengths</span> <span class="o">=</span> <span class="n">words</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">ComputeWordLengthFn</span><span class="p">())</span>
</code></pre></div></div>
@@ -1239,6 +1243,7 @@ method.</p>
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">element</span><span class="p">):</span>
<span class="k">return</span> <span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">element</span><span class="p">)]</span>
+
</code></pre></div></div>
<blockquote class="language-java">
@@ -1545,18 +1550,24 @@ unique key from any of the input collections.</p>
<span class="o">.</span><span class="na">and</span><span class="o">(</span><span class="n">phonesTag</span><span class="o">,</span> <span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">())));</span>
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">results</span> <span class="o">=</span> <span class="p">[</span>
- <span class="p">(</span><span class="s">'amy'</span><span class="p">,</span> <span class="p">{</span>
- <span class="s">'emails'</span><span class="p">:</span> <span class="p">[</span><span class="s">'amy@example.com'</span><span class="p">],</span>
- <span class="s">'phones'</span><span class="p">:</span> <span class="p">[</span><span class="s">'111-222-3333'</span><span class="p">,</span> <span class="s">'333-444-5555'</span><span class="p">]}),</span>
- <span class="p">(</span><span class="s">'carl'</span><span class="p">,</span> <span class="p">{</span>
- <span class="s">'emails'</span><span class="p">:</span> <span class="p">[</span><span class="s">'carl@email.com'</span><span class="p">,</span> <span class="s">'carl@example.com'</span><span class="p">],</span>
- <span class="s">'phones'</span><span class="p">:</span> <span class="p">[</span><span class="s">'444-555-6666'</span><span class="p">]}),</span>
+ <span class="p">(</span>
+ <span class="s">'amy'</span><span class="p">,</span>
+ <span class="p">{</span>
+ <span class="s">'emails'</span><span class="p">:</span> <span class="p">[</span><span class="s">'amy@example.com'</span><span class="p">],</span>
+ <span class="s">'phones'</span><span class="p">:</span> <span class="p">[</span><span class="s">'111-222-3333'</span><span class="p">,</span> <span class="s">'333-444-5555'</span><span class="p">]</span>
+ <span class="p">}),</span>
+ <span class="p">(</span>
+ <span class="s">'carl'</span><span class="p">,</span>
+ <span class="p">{</span>
+ <span class="s">'emails'</span><span class="p">:</span> <span class="p">[</span><span class="s">'carl@email.com'</span><span class="p">,</span> <span class="s">'carl@example.com'</span><span class="p">],</span>
+ <span class="s">'phones'</span><span class="p">:</span> <span class="p">[</span><span class="s">'444-555-6666'</span><span class="p">]</span>
+ <span class="p">}),</span>
<span class="p">(</span><span class="s">'james'</span><span class="p">,</span> <span class="p">{</span>
- <span class="s">'emails'</span><span class="p">:</span> <span class="p">[],</span>
- <span class="s">'phones'</span><span class="p">:</span> <span class="p">[</span><span class="s">'222-333-4444'</span><span class="p">]}),</span>
+ <span class="s">'emails'</span><span class="p">:</span> <span class="p">[],</span> <span class="s">'phones'</span><span class="p">:</span> <span class="p">[</span><span class="s">'222-333-4444'</span><span class="p">]</span>
+ <span class="p">}),</span>
<span class="p">(</span><span class="s">'julia'</span><span class="p">,</span> <span class="p">{</span>
- <span class="s">'emails'</span><span class="p">:</span> <span class="p">[</span><span class="s">'julia@example.com'</span><span class="p">],</span>
- <span class="s">'phones'</span><span class="p">:</span> <span class="p">[]}),</span>
+ <span class="s">'emails'</span><span class="p">:</span> <span class="p">[</span><span class="s">'julia@example.com'</span><span class="p">],</span> <span class="s">'phones'</span><span class="p">:</span> <span class="p">[]</span>
+ <span class="p">}),</span>
<span class="p">]</span>
</code></pre></div></div>
@@ -1590,8 +1601,7 @@ and format data from each collection.</p>
<span class="c"># the value will be a dictionary with two entries: 'emails' - an iterable of</span>
<span class="c"># all values for the current key in the emails PCollection and 'phones': an</span>
<span class="c"># iterable of all values for the current key in the phones PCollection.</span>
-<span class="n">results</span> <span class="o">=</span> <span class="p">({</span><span class="s">'emails'</span><span class="p">:</span> <span class="n">emails</span><span class="p">,</span> <span class="s">'phones'</span><span class="p">:</span> <span class="n">phones</span><span class="p">}</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CoGroupByKey</span><span class="p">())</span>
+<span class="n">results</span> <span class="o">=</span> <span class="p">({</span><span class="s">'emails'</span><span class="p">:</span> <span class="n">emails</span><span class="p">,</span> <span class="s">'phones'</span><span class="p">:</span> <span class="n">phones</span><span class="p">}</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CoGroupByKey</span><span class="p">())</span>
<span class="k">def</span> <span class="nf">join_info</span><span class="p">(</span><span class="n">name_info</span><span class="p">):</span>
<span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">info</span><span class="p">)</span> <span class="o">=</span> <span class="n">name_info</span>
@@ -1662,7 +1672,8 @@ input/output type.</p>
<span class="k">def</span> <span class="nf">bounded_sum</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">bound</span><span class="o">=</span><span class="mi">500</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">min</span><span class="p">(</span><span class="nb">sum</span><span class="p">(</span><span class="n">values</span><span class="p">),</span> <span class="n">bound</span><span class="p">)</span>
-<span class="n">small_sum</span> <span class="o">=</span> <span class="n">pc</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombineGlobally</span><span class="p">(</span><span class="n">bounded_sum</span><span class="p">)</span> <span class="c"># [500]</span>
+
+<span class="n">small_sum</span> <span class="o">=</span> <span class="n">pc</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombineGlobally</span><span class="p">(</span><span class="n">bounded_sum</span><span class="p">)</span> <span class="c"># [500]</span>
<span class="n">large_sum</span> <span class="o">=</span> <span class="n">pc</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombineGlobally</span><span class="p">(</span><span class="n">bounded_sum</span><span class="p">,</span> <span class="n">bound</span><span class="o">=</span><span class="mi">5000</span><span class="p">)</span> <span class="c"># [1111]</span>
</code></pre></div></div>
@@ -1755,6 +1766,7 @@ mean average:</p>
<span class="k">def</span> <span class="nf">extract_output</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sum_count</span><span class="p">):</span>
<span class="p">(</span><span class="nb">sum</span><span class="p">,</span> <span class="n">count</span><span class="p">)</span> <span class="o">=</span> <span class="n">sum_count</span>
<span class="k">return</span> <span class="nb">sum</span> <span class="o">/</span> <span class="n">count</span> <span class="k">if</span> <span class="n">count</span> <span class="k">else</span> <span class="nb">float</span><span class="p">(</span><span class="s">'NaN'</span><span class="p">)</span>
+
</code></pre></div></div>
<h5 id="combining-pcollection">4.2.4.3. Combining a PCollection into a single value</h5>
@@ -1854,9 +1866,9 @@ PerKey must be an associative reduction function or a subclass of <code class="h
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># PCollection is grouped by key and the numeric values associated with each key</span>
<span class="c"># are averaged into a float.</span>
<span class="n">player_accuracies</span> <span class="o">=</span> <span class="o">...</span>
-<span class="n">avg_accuracy_per_player</span> <span class="o">=</span> <span class="p">(</span><span class="n">player_accuracies</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombinePerKey</span><span class="p">(</span>
- <span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">MeanCombineFn</span><span class="p">()))</span>
+<span class="n">avg_accuracy_per_player</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">player_accuracies</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombinePerKey</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">MeanCombineFn</span><span class="p">()))</span>
</code></pre></div></div>
@@ -2087,17 +2099,19 @@ determined by the input data, or depend on a different branch of your pipeline.<
<span class="k">yield</span> <span class="n">word</span>
<span class="c"># Construct a deferred side input.</span>
-<span class="n">avg_word_len</span> <span class="o">=</span> <span class="p">(</span><span class="n">words</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="nb">len</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombineGlobally</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">MeanCombineFn</span><span class="p">()))</span>
+<span class="n">avg_word_len</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">words</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="nb">len</span><span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombineGlobally</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">MeanCombineFn</span><span class="p">()))</span>
<span class="c"># Call with explicit side inputs.</span>
<span class="n">small_words</span> <span class="o">=</span> <span class="n">words</span> <span class="o">|</span> <span class="s">'small'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span><span class="n">filter_using_length</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="c"># A single deferred side input.</span>
-<span class="n">larger_than_average</span> <span class="o">=</span> <span class="p">(</span><span class="n">words</span> <span class="o">|</span> <span class="s">'large'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span>
- <span class="n">filter_using_length</span><span class="p">,</span>
- <span class="n">lower_bound</span><span class="o">=</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsSingleton</span><span class="p">(</span><span class="n">avg_word_len</span><span class="p">)))</span>
+<span class="n">larger_than_average</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">words</span> <span class="o">|</span> <span class="s">'large'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span>
+ <span class="n">filter_using_length</span><span class="p">,</span> <span class="n">lower_bound</span><span class="o">=</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsSingleton</span><span class="p">(</span><span class="n">avg_word_len</span><span class="p">))</span>
+<span class="p">)</span>
<span class="c"># Mix and match.</span>
<span class="n">small_but_nontrivial</span> <span class="o">=</span> <span class="n">words</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span>
@@ -2210,9 +2224,12 @@ together.</p>
<span class="c"># with_outputs are attributes on the returned DoOutputsTuple object. The tags give access to the</span>
<span class="c"># corresponding output PCollections.</span>
-<span class="n">results</span> <span class="o">=</span> <span class="p">(</span><span class="n">words</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">ProcessWords</span><span class="p">(),</span> <span class="n">cutoff_length</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s">'x'</span><spa [...]
- <span class="o">.</span><span class="n">with_outputs</span><span class="p">(</span><span class="s">'above_cutoff_lengths'</span><span class="p">,</span> <span class="s">'marked strings'</span><span class="p">,</span>
- <span class="n">main</span><span class="o">=</span><span class="s">'below_cutoff_strings'</span><span class="p">))</span>
+<span class="n">results</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">words</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">ProcessWords</span><span class="p">(),</span> <span class="n">cutoff_length</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s">'x'</span><span class="p">)</span><span class="o">.</span><span class="n">with_outputs</span><span class="p">(</span>
+ <span class="s">'above_cutoff_lengths'</span><span class="p">,</span>
+ <span class="s">'marked strings'</span><span class="p">,</span>
+ <span class="n">main</span><span class="o">=</span><span class="s">'below_cutoff_strings'</span><span class="p">))</span>
<span class="n">below</span> <span class="o">=</span> <span class="n">results</span><span class="o">.</span><span class="n">below_cutoff_strings</span>
<span class="n">above</span> <span class="o">=</span> <span class="n">results</span><span class="o">.</span><span class="n">above_cutoff_lengths</span>
<span class="n">marked</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s">'marked strings'</span><span class="p">]</span> <span class="c"># indexing works as well</span>
@@ -2258,20 +2275,19 @@ together.</p>
<span class="c"># Based on the previous example, this shows the DoFn emitting to the main output and two additional outputs.</span>
<span class="k">class</span> <span class="nc">ProcessWords</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="p">):</span>
-
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">element</span><span class="p">,</span> <span class="n">cutoff_length</span><span class="p">,</span> <span class="n">marker</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">element</span><span class="p">)</span> <span class="o"><=</span> <span class="n">cutoff_length</span><span class="p">:</span>
<span class="c"># Emit this short word to the main output.</span>
<span class="k">yield</span> <span class="n">element</span>
<span class="k">else</span><span class="p">:</span>
<span class="c"># Emit this word's long length to the 'above_cutoff_lengths' output.</span>
- <span class="k">yield</span> <span class="n">pvalue</span><span class="o">.</span><span class="n">TaggedOutput</span><span class="p">(</span>
- <span class="s">'above_cutoff_lengths'</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">element</span><span class="p">))</span>
+ <span class="k">yield</span> <span class="n">pvalue</span><span class="o">.</span><span class="n">TaggedOutput</span><span class="p">(</span><span class="s">'above_cutoff_lengths'</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">element</span><span class="p">))</span>
<span class="k">if</span> <span class="n">element</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">marker</span><span class="p">):</span>
<span class="c"># Emit this word to a different output with the 'marked strings' tag.</span>
<span class="k">yield</span> <span class="n">pvalue</span><span class="o">.</span><span class="n">TaggedOutput</span><span class="p">(</span><span class="s">'marked strings'</span><span class="p">,</span> <span class="n">element</span><span class="p">)</span>
+
<span class="c"># Producing multiple outputs is also available in Map and FlatMap.</span>
<span class="c"># Here is an example that uses FlatMap and shows that the tags do not need to be specified ahead of time.</span>
@@ -2501,15 +2517,16 @@ transform’s intermediate data changes type multiple times.</p>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># The CountWords Composite Transform inside the WordCount pipeline.</span>
<span class="k">class</span> <span class="nc">CountWords</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">PTransform</span><span class="p">):</span>
-
<span class="k">def</span> <span class="nf">expand</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pcoll</span><span class="p">):</span>
- <span class="k">return</span> <span class="p">(</span><span class="n">pcoll</span>
- <span class="c"># Convert lines of text into individual words.</span>
- <span class="o">|</span> <span class="s">'ExtractWords'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">ExtractWordsFn</span><span class="p">())</span>
- <span class="c"># Count the number of times each word occurs.</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">Count</span><span class="o">.</span><span class="n">PerElement</span><span class="p">()</span>
- <span class="c"># Format each word and count into a printable string.</span>
- <span class="o">|</span> <span class="s">'FormatCounts'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">FormatCountsFn</span><span class="p">()))</span>
+ <span class="k">return</span> <span class="p">(</span>
+ <span class="n">pcoll</span>
+ <span class="c"># Convert lines of text into individual words.</span>
+ <span class="o">|</span> <span class="s">'ExtractWords'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">ExtractWordsFn</span><span class="p">())</span>
+ <span class="c"># Count the number of times each word occurs.</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">Count</span><span class="o">.</span><span class="n">PerElement</span><span class="p">()</span>
+ <span class="c"># Format each word and count into a printable string.</span>
+ <span class="o">|</span> <span class="s">'FormatCounts'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">FormatCountsFn</span><span class="p">()))</span>
+
</code></pre></div></div>
<h4 id="composite-transform-creation">4.6.2. Creating a composite transform</h4>
@@ -2537,6 +2554,7 @@ of the multi-collection types for the relevant type parameter.</p>
<span class="k">def</span> <span class="nf">expand</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pcoll</span><span class="p">):</span>
<span class="c"># Transform logic goes here.</span>
<span class="k">return</span> <span class="n">pcoll</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
+
</code></pre></div></div>
<p>Within your <code class="highlighter-rouge">PTransform</code> subclass, you’ll need to override the <code class="highlighter-rouge">expand</code> method.
@@ -2562,6 +2580,7 @@ value.</p>
<span class="k">def</span> <span class="nf">expand</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pcoll</span><span class="p">):</span>
<span class="c"># Transform logic goes here.</span>
<span class="k">return</span> <span class="n">pcoll</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
+
</code></pre></div></div>
<p>As long as you override the <code class="highlighter-rouge">expand</code> method in your <code class="highlighter-rouge">PTransform</code> subclass to
@@ -3321,7 +3340,6 @@ with a <code class="highlighter-rouge">DoFn</code> to attach the timestamps to e
<span class="o">}));</span>
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">class</span> <span class="nc">AddTimestampDoFn</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="p">):</span>
-
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">element</span><span class="p">):</span>
<span class="c"># Extract the numeric Unix seconds-since-epoch timestamp to be</span>
<span class="c"># associated with the current log entry.</span>
@@ -3424,8 +3442,7 @@ firings:</p>
<span class="o">.</span><span class="na">withLateFirings</span><span class="o">(</span><span class="n">AfterPane</span><span class="o">.</span><span class="na">elementCountAtLeast</span><span class="o">(</span><span class="mi">1</span><span class="o">))</span>
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">AfterWatermark</span><span class="p">(</span>
- <span class="n">early</span><span class="o">=</span><span class="n">AfterProcessingTime</span><span class="p">(</span><span class="n">delay</span><span class="o">=</span><span class="mi">1</span> <span class="o">*</span> <span class="mi">60</span><span class="p">),</span>
- <span class="n">late</span><span class="o">=</span><span class="n">AfterCount</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
+ <span class="n">early</span><span class="o">=</span><span class="n">AfterProcessingTime</span><span class="p">(</span><span class="n">delay</span><span class="o">=</span><span class="mi">1</span> <span class="o">*</span> <span class="mi">60</span><span class="p">),</span> <span class="n">late</span><span class="o">=</span><span class="n">AfterCount</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
</code></pre></div></div>
<h4 id="default-trigger">8.1.1. Default trigger</h4>
@@ -3666,8 +3683,7 @@ trigger stops executing</li>
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">pcollection</span> <span class="o">|</span> <span class="n">WindowInto</span><span class="p">(</span>
<span class="n">FixedWindows</span><span class="p">(</span><span class="mi">1</span> <span class="o">*</span> <span class="mi">60</span><span class="p">),</span>
- <span class="n">trigger</span><span class="o">=</span><span class="n">AfterWatermark</span><span class="p">(</span>
- <span class="n">late</span><span class="o">=</span><span class="n">AfterProcessingTime</span><span class="p">(</span><span class="mi">10</span> <span class="o">*</span> <span class="mi">60</span><span class="p">)),</span>
+ <span class="n">trigger</span><span class="o">=</span><span class="n">AfterWatermark</span><span class="p">(</span><span class="n">late</span><span class="o">=</span><span class="n">AfterProcessingTime</span><span class="p">(</span><span class="mi">10</span> <span class="o">*</span> <span class="mi">60</span><span class="p">)),</span>
<span class="n">allowed_lateness</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
<span class="n">accumulation_mode</span><span class="o">=</span><span class="n">AccumulationMode</span><span class="o">.</span><span class="n">DISCARDING</span><span class="p">)</span>
</code></pre></div></div>
@@ -3685,9 +3701,7 @@ elements, or after a minute.</p>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">pcollection</span> <span class="o">|</span> <span class="n">WindowInto</span><span class="p">(</span>
<span class="n">FixedWindows</span><span class="p">(</span><span class="mi">1</span> <span class="o">*</span> <span class="mi">60</span><span class="p">),</span>
<span class="n">trigger</span><span class="o">=</span><span class="n">Repeatedly</span><span class="p">(</span>
- <span class="n">AfterAny</span><span class="p">(</span>
- <span class="n">AfterCount</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span>
- <span class="n">AfterProcessingTime</span><span class="p">(</span><span class="mi">1</span> <span class="o">*</span> <span class="mi">60</span><span class="p">))),</span>
+ <span class="n">AfterAny</span><span class="p">(</span><span class="n">AfterCount</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">AfterProcessingTime</span><span class="p">(</span><span class="mi">1</span> <span class="o">*</span> <span class="mi">60</span><span class="p">))),</span>
<span class="n">accumulation_mode</span><span class="o">=</span><span class="n">AccumulationMode</span><span class="o">.</span><span class="n">DISCARDING</span><span class="p">)</span>
</code></pre></div></div>
diff --git a/website/generated-content/documentation/sdks/python-type-safety/index.html b/website/generated-content/documentation/sdks/python-type-safety/index.html
index 8558ae3..801ee48 100644
--- a/website/generated-content/documentation/sdks/python-type-safety/index.html
+++ b/website/generated-content/documentation/sdks/python-type-safety/index.html
@@ -381,6 +381,7 @@ class FilterEvensDoFn(beam.DoFn):
def process(self, element):
if element % 2 == 0:
yield element
+
evens = numbers | beam.ParDo(FilterEvensDoFn())
</code></pre></div></div>
@@ -502,8 +503,7 @@ def parse_player_and_score(csv):
totals = (
lines
| beam.Map(parse_player_and_score)
- | beam.CombinePerKey(sum).with_input_types(
- typing.Tuple[Player, int]))
+ | beam.CombinePerKey(sum).with_input_types(typing.Tuple[Player, int]))
</code></pre></div></div>
</div>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/filter/index.html b/website/generated-content/documentation/transforms/python/elementwise/filter/index.html
index 008c03e..50d1570 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/filter/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/filter/index.html
@@ -558,15 +558,24 @@ Then, we apply <code class="highlighter-rouge">Filter</code> in multiple ways to
<span class="n">perennials</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Gardening plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Filter perennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span><span class="n">is_perennial</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Filter</code>:</p>
@@ -642,16 +651,25 @@ limitations under the License.
<span class="n">perennials</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Gardening plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
<span class="p">])</span>
- <span class="o">|</span> <span class="s">'Filter perennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">plant</span><span class="p">:</span> <span class="n">plant</span><span class="p">[</span><span class="s">'duration'</span><span class="p">]</span> <span class="o">==</span> <span class="s">'perennial'</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="s">'Filter perennials'</span> <span class="o">>></span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">plant</span><span class="p">:</span> <span class="n">plant</span><span class="p">[</span><span class="s">'duration'</span><span class="p">]</span> <span class="o">==</span> <span class="s">'perennial'</span><span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Filter</code>:</p>
@@ -733,15 +751,24 @@ They are passed as additional positional arguments or keyword arguments to the f
<span class="n">perennials</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Gardening plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Filter perennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span><span class="n">has_duration</span><span class="p">,</span> <span class="s">'perennial'</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Filter</code>:</p>
@@ -823,18 +850,28 @@ We then use that value to filter out perennials.</p>
<span class="n">perennials</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Gardening plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Filter perennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">plant</span><span class="p">,</span> <span class="n">duration</span><span class="p">:</span> <span class="n">plant</span><span class="p">[</span><span class="s">'duration'</span><span class="p">]</span> <span class="o">==</span> <span class="n">duration</span><span class="p">,</span>
+ <span class="k">lambda</span> <span class="n">plant</span><span class="p">,</span>
+ <span class="n">duration</span><span class="p">:</span> <span class="n">plant</span><span class="p">[</span><span class="s">'duration'</span><span class="p">]</span> <span class="o">==</span> <span class="n">duration</span><span class="p">,</span>
<span class="n">duration</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsSingleton</span><span class="p">(</span><span class="n">perennial</span><span class="p">),</span>
<span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Filter</code>:</p>
@@ -918,18 +955,28 @@ so it is possible to iterate over large <code class="highlighter-rouge">PCollect
<span class="n">valid_plants</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Gardening plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'PERENNIAL'</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'PERENNIAL'</span>
+ <span class="p">},</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Filter valid plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">plant</span><span class="p">,</span> <span class="n">valid_durations</span><span class="p">:</span> <span class="n">plant</span><span class="p">[</span><span class="s">'duration'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">valid_durations</span><span class="p">,</span>
+ <span class="k">lambda</span> <span class="n">plant</span><span class="p">,</span>
+ <span class="n">valid_durations</span><span class="p">:</span> <span class="n">plant</span><span class="p">[</span><span class="s">'duration'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">valid_durations</span><span class="p">,</span>
<span class="n">valid_durations</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsIter</span><span class="p">(</span><span class="n">valid_durations</span><span class="p">),</span>
<span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Filter</code>:</p>
@@ -1020,18 +1067,28 @@ If the <code class="highlighter-rouge">PCollection</code> won’t fit into memor
<span class="n">perennials</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Gardening plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'biennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Filter plants by duration'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">plant</span><span class="p">,</span> <span class="n">keep_duration</span><span class="p">:</span> <span class="n">keep_duration</span><span class="p">[</span><span class="n">plant</span><span class="p">[</span><span class="s">'duration'</span><span class="p">]],</span>
+ <span class="k">lambda</span> <span class="n">plant</span><span class="p">,</span>
+ <span class="n">keep_duration</span><span class="p">:</span> <span class="n">keep_duration</span><span class="p">[</span><span class="n">plant</span><span class="p">[</span><span class="s">'duration'</span><span class="p">]],</span>
<span class="n">keep_duration</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsDict</span><span class="p">(</span><span class="n">keep_duration</span><span class="p">),</span>
<span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Filter</code>:</p>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html b/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html
index b0a1d60..4aacf4c 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html
@@ -563,8 +563,7 @@ This pipeline splits the input element using whitespaces, creating a list of zer
<span class="s">'🍅Tomato 🥔Potato'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Split words'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span><span class="nb">str</span><span class="o">.</span><span class="n">split</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">FlatMap</code>:</p>
@@ -649,8 +648,7 @@ limitations under the License.
<span class="s">'🍅Tomato,🥔Potato'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Split words'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span><span class="n">split_words</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">FlatMap</code>:</p>
@@ -734,8 +732,7 @@ We use a lambda function that returns the same input element it received.</p>
<span class="p">[</span><span class="s">'🍅Tomato'</span><span class="p">,</span> <span class="s">'🥔Potato'</span><span class="p">],</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Flatten lists'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span><span class="k">lambda</span> <span class="n">elements</span><span class="p">:</span> <span class="n">elements</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">FlatMap</code>:</p>
@@ -823,8 +820,7 @@ Each yielded result in the generator is an element in the resulting <code class=
<span class="p">[</span><span class="s">'🍅Tomato'</span><span class="p">,</span> <span class="s">'🥔Potato'</span><span class="p">],</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Flatten lists'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span><span class="n">generate_elements</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">FlatMap</code>:</p>
@@ -915,8 +911,7 @@ you can use <code class="highlighter-rouge">FlatMapTuple</code> to unpack them i
<span class="p">(</span><span class="bp">None</span><span class="p">,</span> <span class="s">'Invalid'</span><span class="p">),</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Format'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMapTuple</span><span class="p">(</span><span class="n">format_plant</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">FlatMapTuple</code>:</p>
@@ -1004,8 +999,7 @@ They are passed as additional positional arguments or keyword arguments to the f
<span class="s">'🍅Tomato,🥔Potato'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Split words'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span><span class="n">split_words</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s">','</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">FlatMap</code>:</p>
@@ -1093,11 +1087,11 @@ We then use that value as the delimiter for the <code class="highlighter-rouge">
<span class="s">'🍅Tomato,🥔Potato'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Split words'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">text</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">:</span> <span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">delimiter</span><span class="p">),</span>
+ <span class="k">lambda</span> <span class="n">text</span><span class="p">,</span>
+ <span class="n">delimiter</span><span class="p">:</span> <span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">delimiter</span><span class="p">),</span>
<span class="n">delimiter</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsSingleton</span><span class="p">(</span><span class="n">delimiter</span><span class="p">),</span>
<span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">FlatMap</code>:</p>
@@ -1188,18 +1182,27 @@ so it is possible to iterate over large <code class="highlighter-rouge">PCollect
<span class="n">valid_plants</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Gardening plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'Perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'BIENNIAL'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'unknown'</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'Perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'BIENNIAL'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'perennial'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'annual'</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="s">'unknown'</span>
+ <span class="p">},</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Normalize and validate durations'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span>
<span class="n">normalize_and_validate_durations</span><span class="p">,</span>
<span class="n">valid_durations</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsIter</span><span class="p">(</span><span class="n">valid_durations</span><span class="p">),</span>
<span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">FlatMap</code>:</p>
@@ -1295,18 +1298,27 @@ If the <code class="highlighter-rouge">PCollection</code> won’t fit into memor
<span class="n">valid_plants</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Gardening plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">0</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="o">-</span><span class="mi">1</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">1</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">0</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="o">-</span><span class="mi">1</span>
+ <span class="p">},</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Replace duration if valid'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span>
<span class="n">replace_duration_if_valid</span><span class="p">,</span>
<span class="n">durations</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsDict</span><span class="p">(</span><span class="n">durations</span><span class="p">),</span>
<span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">FlatMap</code>:</p>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/keys/index.html b/website/generated-content/documentation/transforms/python/elementwise/keys/index.html
index eb0dcec..f02196c 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/keys/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/keys/index.html
@@ -545,8 +545,7 @@ Then, we apply <code class="highlighter-rouge">Keys</code> to extract the keys a
<span class="p">(</span><span class="s">'🥔'</span><span class="p">,</span> <span class="s">'Potato'</span><span class="p">),</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Keys'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Keys</span><span class="p">()</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Keys</code>:</p>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html b/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html
index 69e93f8..2542b80 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html
@@ -546,8 +546,7 @@ Then, we apply <code class="highlighter-rouge">KvSwap</code> to swap the keys an
<span class="p">(</span><span class="s">'🥔'</span><span class="p">,</span> <span class="s">'Potato'</span><span class="p">),</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Key-Value swap'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">KvSwap</span><span class="p">()</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">KvSwap</code>:</p>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/map/index.html b/website/generated-content/documentation/transforms/python/elementwise/map/index.html
index 39d255a..cff16c2 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/map/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/map/index.html
@@ -563,8 +563,7 @@ It strips the input element’s whitespaces, including newlines and tabs.</p>
<span class="s">' 🥔Potato </span><span class="se">\n</span><span class="s">'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Strip'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="nb">str</span><span class="o">.</span><span class="n">strip</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Map</code>:</p>
@@ -652,8 +651,7 @@ limitations under the License.
<span class="s">'# 🥔Potato</span><span class="se">\n</span><span class="s">'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Strip header'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="n">strip_header_and_newline</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Map</code>:</p>
@@ -738,8 +736,7 @@ limitations under the License.
<span class="s">'# 🥔Potato</span><span class="se">\n</span><span class="s">'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Strip header'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">text</span><span class="p">:</span> <span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s">'# </span><span class="se">\n</span><span class="s">'</span><span class="p">))</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Map</code>:</p>
@@ -830,8 +827,7 @@ They are passed as additional positional arguments or keyword arguments to the f
<span class="s">'# 🥔Potato</span><span class="se">\n</span><span class="s">'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Strip header'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="n">strip</span><span class="p">,</span> <span class="n">chars</span><span class="o">=</span><span class="s">'# </span><span class="se">\n</span><span class="s">'</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Map</code>:</p>
@@ -916,10 +912,9 @@ you can use <code class="highlighter-rouge">MapTuple</code> to unpack them into
<span class="p">(</span><span class="s">'🍅'</span><span class="p">,</span> <span class="s">'Tomato'</span><span class="p">),</span>
<span class="p">(</span><span class="s">'🥔'</span><span class="p">,</span> <span class="s">'Potato'</span><span class="p">),</span>
<span class="p">])</span>
- <span class="o">|</span> <span class="s">'Format'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">MapTuple</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">icon</span><span class="p">,</span> <span class="n">plant</span><span class="p">:</span> <span class="s">'{}{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">icon</span><span class="p">,</span> <span class="n">plant</span><span class="p">))</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="s">'Format'</span> <span class="o">>></span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">MapTuple</span><span class="p">(</span><span class="k">lambda</span> <span class="n">icon</span><span class="p">,</span> <span class="n">plant</span><span class="p">:</span> <span class="s">'{}{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">icon</span><span class="p">,</span> <span class="n">plant</span><span class="p">))</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">MapTuple</code>:</p>
@@ -1010,11 +1005,11 @@ We then use that value as the characters for the <code class="highlighter-rouge"
<span class="s">'# 🥔Potato</span><span class="se">\n</span><span class="s">'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Strip header'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">text</span><span class="p">,</span> <span class="n">chars</span><span class="p">:</span> <span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="n">chars</span><span class="p">),</span>
+ <span class="k">lambda</span> <span class="n">text</span><span class="p">,</span>
+ <span class="n">chars</span><span class="p">:</span> <span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="n">chars</span><span class="p">),</span>
<span class="n">chars</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsSingleton</span><span class="p">(</span><span class="n">chars</span><span class="p">),</span>
<span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Map</code>:</p>
@@ -1103,11 +1098,11 @@ so it is possible to iterate over large <code class="highlighter-rouge">PCollect
<span class="s">'# 🥔Potato</span><span class="se">\n</span><span class="s">'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Strip header'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">text</span><span class="p">,</span> <span class="n">chars</span><span class="p">:</span> <span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">chars</span><span class="p">)),</span>
+ <span class="k">lambda</span> <span class="n">text</span><span class="p">,</span>
+ <span class="n">chars</span><span class="p">:</span> <span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">chars</span><span class="p">)),</span>
<span class="n">chars</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsIter</span><span class="p">(</span><span class="n">chars</span><span class="p">),</span>
<span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Map</code>:</p>
@@ -1203,18 +1198,27 @@ If the <code class="highlighter-rouge">PCollection</code> won’t fit into memor
<span class="n">plant_details</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Gardening plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">0</span><span class="p">},</span>
- <span class="p">{</span><span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span><span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍓'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Strawberry'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Carrot'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">1</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Eggplant'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Tomato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">0</span>
+ <span class="p">},</span>
+ <span class="p">{</span>
+ <span class="s">'icon'</span><span class="p">:</span> <span class="s">'🥔'</span><span class="p">,</span> <span class="s">'name'</span><span class="p">:</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'duration'</span><span class="p">:</span> <span class="mi">2</span>
+ <span class="p">},</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Replace duration'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span>
<span class="n">replace_duration</span><span class="p">,</span>
<span class="n">durations</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">pvalue</span><span class="o">.</span><span class="n">AsDict</span><span class="p">(</span><span class="n">durations</span><span class="p">),</span>
<span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Map</code>:</p>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html b/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html
index 6c8e749..798cceb 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html
@@ -569,8 +569,7 @@ and it can yield zero or more output elements.</p>
<span class="s">'🍅Tomato,🥔Potato'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Split words'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">SplitWords</span><span class="p">(</span><span class="s">','</span><span class="p">))</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">ParDo</code>:</p>
@@ -656,7 +655,11 @@ object.</li>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="kn">import</span> <span class="nn">apache_beam</span> <span class="k">as</span> <span class="n">beam</span>
<span class="k">class</span> <span class="nc">AnalyzeElement</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="p">):</span>
- <span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">elem</span><span class="p">,</span> <span class="n">timestamp</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="o">.</span><span class="n">TimestampParam</span><span class="p">,</span> <span class="n">window</span><span class="o">=</span><span class="n">beam</span><sp [...]
+ <span class="k">def</span> <span class="nf">process</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">elem</span><span class="p">,</span>
+ <span class="n">timestamp</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="o">.</span><span class="n">TimestampParam</span><span class="p">,</span>
+ <span class="n">window</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="o">.</span><span class="n">WindowParam</span><span class="p">):</span>
<span class="k">yield</span> <span class="s">'</span><span class="se">\n</span><span class="s">'</span><span class="o">.</span><span class="n">join</span><span class="p">([</span>
<span class="s">'# timestamp'</span><span class="p">,</span>
<span class="s">'type(timestamp) -> '</span> <span class="o">+</span> <span class="nb">repr</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)),</span>
@@ -666,21 +669,24 @@ object.</li>
<span class="s">''</span><span class="p">,</span>
<span class="s">'# window'</span><span class="p">,</span>
<span class="s">'type(window) -> '</span> <span class="o">+</span> <span class="nb">repr</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">window</span><span class="p">)),</span>
- <span class="s">'window.start -> {} ({})'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">window</span><span class="o">.</span><span class="n">start</span><span class="p">,</span> <span class="n">window</span><span class="o">.</span><span class="n">start</span><span class="o">.</span><span class="n">to_utc_datetime</span><span class="p">()),</span>
- <span class="s">'window.end -> {} ({})'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">window</span><span class="o">.</span><span class="n">end</span><span class="p">,</span> <span class="n">window</span><span class="o">.</span><span class="n">end</span><span class="o">.</span><span class="n">to_utc_datetime</span><span class="p">()),</span>
- <span class="s">'window.max_timestamp() -> {} ({})'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">window</span><span class="o">.</span><span class="n">max_timestamp</span><span class="p">(),</span> <span class="n">window</span><span class="o">.</span><span class="n">max_timestamp</span><span class="p">()</span><span class="o">.</span><span class="n">to_utc_datetime</span><span class="p">()),</span>
+ <span class="s">'window.start -> {} ({})'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">window</span><span class="o">.</span><span class="n">start</span><span class="p">,</span> <span class="n">window</span><span class="o">.</span><span class="n">start</span><span class="o">.</span><span class="n">to_utc_datetime</span><span class="p">()),</span>
+ <span class="s">'window.end -> {} ({})'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">window</span><span class="o">.</span><span class="n">end</span><span class="p">,</span> <span class="n">window</span><span class="o">.</span><span class="n">end</span><span class="o">.</span><span class="n">to_utc_datetime</span><span class="p">()),</span>
+ <span class="s">'window.max_timestamp() -> {} ({})'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">window</span><span class="o">.</span><span class="n">max_timestamp</span><span class="p">(),</span> <span class="n">window</span><span class="o">.</span><span class="n">max_timestamp</span><span class="p">()</span><span class="o">.</span><span class="n">to_utc_datetime</span><span class="p">()),</span>
<span class="p">])</span>
<span class="k">with</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">()</span> <span class="k">as</span> <span class="n">pipeline</span><span class="p">:</span>
<span class="n">dofn_params</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Create a single test element'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="s">':)'</span><span class="p">])</span>
- <span class="o">|</span> <span class="s">'Add timestamp (Spring equinox 2020)'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">TimestampedValue</span><span class="p">(</span><span class="n">elem</span><span class="p">,</span> <span class="mi">1584675660</span><span class="p">))</span>
- <span class="o">|</span> <span class="s">'Fixed 30sec windows'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">WindowInto</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">FixedWindows</span><span class="p">(</span><span class="mi">30</span><span class="p">))</span>
+ <span class="o">|</span> <span class="s">'Add timestamp (Spring equinox 2020)'</span> <span class="o">>></span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">TimestampedValue</span><span class="p">(</span><span class="n">elem</span><span class="p">,</span> <span class="mi">1584675660</span><span class="p">))</span>
+ <span class="o">|</span>
+ <span class="s">'Fixed 30sec windows'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">WindowInto</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">FixedWindows</span><span class="p">(</span><span class="mi">30</span><span class="p">))</span>
<span class="o">|</span> <span class="s">'Analyze element'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">AnalyzeElement</span><span class="p">())</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip"><code class="highlighter-rouge">stdout</code> output:</p>
@@ -839,8 +845,7 @@ For example, if the worker crashes, <code class="highlighter-rouge">teardown</co
<span class="n">pipeline</span>
<span class="o">|</span> <span class="s">'Create inputs'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="s">'🍓'</span><span class="p">,</span> <span class="s">'🥕'</span><span class="p">,</span> <span class="s">'🍆'</span><span class="p">,</span> <span class="s">'🍅'</span><span class="p">,</span> <span class="s">'🥔'</span><span class="p">])</span>
<span class="o">|</span> <span class="s">'DoFn methods'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">DoFnMethods</span><span class="p">())</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip"><code class="highlighter-rouge">stdout</code> output:</p>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html b/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
index 8c843db..b6596d2 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
@@ -577,8 +577,10 @@ We partition the <code class="highlighter-rouge">PCollection</code> into one <co
<span class="p">)</span>
<span class="n">annuals</span> <span class="o">|</span> <span class="s">'Annuals'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'annual: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</span>< [...]
- <span class="n">biennials</span> <span class="o">|</span> <span class="s">'Biennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'biennial: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</ [...]
- <span class="n">perennials</span> <span class="o">|</span> <span class="s">'Perennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'perennial: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n"> [...]
+ <span class="n">biennials</span> <span class="o">|</span> <span class="s">'Biennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span>
+ <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'biennial: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</span><span class="p">)))</span>
+ <span class="n">perennials</span> <span class="o">|</span> <span class="s">'Perennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span>
+ <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'perennial: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</span><span class="p">)))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code>s:</p>
@@ -671,8 +673,10 @@ limitations under the License.
<span class="p">)</span>
<span class="n">annuals</span> <span class="o">|</span> <span class="s">'Annuals'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'annual: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</span>< [...]
- <span class="n">biennials</span> <span class="o">|</span> <span class="s">'Biennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'biennial: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</ [...]
- <span class="n">perennials</span> <span class="o">|</span> <span class="s">'Perennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'perennial: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n"> [...]
+ <span class="n">biennials</span> <span class="o">|</span> <span class="s">'Biennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span>
+ <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'biennial: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</span><span class="p">)))</span>
+ <span class="n">perennials</span> <span class="o">|</span> <span class="s">'Perennials'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span>
+ <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'perennial: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</span><span class="p">)))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code>s:</p>
@@ -799,7 +803,7 @@ You might want to adapt the bucket assignment to use a more appropriate or rando
<span class="p">)</span>
<span class="n">train_dataset</span> <span class="o">|</span> <span class="s">'Train'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'train: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</spa [...]
- <span class="n">test_dataset</span> <span class="o">|</span> <span class="s">'Test'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'test: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</span> [...]
+ <span class="n">test_dataset</span> <span class="o">|</span> <span class="s">'Test'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s">'test: {}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">x</span>< [...]
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code>s:</p>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/regex/index.html b/website/generated-content/documentation/transforms/python/elementwise/regex/index.html
index 63bd8d9..0d154d6 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/regex/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/regex/index.html
@@ -596,8 +596,7 @@ To match until the end of the string, add <code class="highlighter-rouge">'$'</c
<span class="s">'invalid, 🍉, format'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Parse plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Regex</span><span class="o">.</span><span class="n">matches</span><span class="p">(</span><span class="n">regex</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Regex.matches</code>:</p>
@@ -695,8 +694,7 @@ To match until the end of the string, add <code class="highlighter-rouge">'$'</c
<span class="s">'invalid, 🍉, format'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Parse plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Regex</span><span class="o">.</span><span class="n">all_matches</span><span class="p">(</span><span class="n">regex</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Regex.all_matches</code>:</p>
@@ -795,8 +793,7 @@ To match until the end of the string, add <code class="highlighter-rouge">'$'</c
<span class="s">'invalid, 🍉, format'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Parse plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Regex</span><span class="o">.</span><span class="n">matches_kv</span><span class="p">(</span><span class="n">regex</span><span class="p">,</span> <span class="n">keyGroup</span><span class="o">=</span><span class="s">'icon'</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Regex.matches_kv</code>:</p>
@@ -893,8 +890,7 @@ To match until the end of the string, add <code class="highlighter-rouge">'$'</c
<span class="s">'# 🥔, Potato, perennial'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Parse plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Regex</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">regex</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Regex.find</code>:</p>
@@ -991,8 +987,7 @@ To match until the end of the string, add <code class="highlighter-rouge">'$'</c
<span class="s">'# 🥔, Potato, perennial'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Parse plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Regex</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="n">regex</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Regex.find_all</code>:</p>
@@ -1090,8 +1085,7 @@ To match until the end of the string, add <code class="highlighter-rouge">'$'</c
<span class="s">'# 🥔, Potato, perennial'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Parse plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Regex</span><span class="o">.</span><span class="n">find_kv</span><span class="p">(</span><span class="n">regex</span><span class="p">,</span> <span class="n">keyGroup</span><span class="o">=</span><span class="s">'icon'</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Regex.find_kv</code>:</p>
@@ -1181,8 +1175,7 @@ on the <code class="highlighter-rouge">replacement</code>.</p>
<span class="s">'🥔 : Potato : perennial'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'To CSV'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Regex</span><span class="o">.</span><span class="n">replace_all</span><span class="p">(</span><span class="s">r'\s*:\s*'</span><span class="p">,</span> <span class="s">','</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Regex.replace_all</code>:</p>
@@ -1270,8 +1263,7 @@ on the <code class="highlighter-rouge">replacement</code>.</p>
<span class="s">'🥔, Potato, perennial'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'As dictionary'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Regex</span><span class="o">.</span><span class="n">replace_first</span><span class="p">(</span><span class="s">r'\s*,\s*'</span><span class="p">,</span> <span class="s">': '</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Regex.replace_first</code>:</p>
@@ -1357,8 +1349,7 @@ The argument <code class="highlighter-rouge">outputEmpty</code> is set to <code
<span class="s">'🥔 : Potato : perennial'</span><span class="p">,</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Parse plants'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Regex</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">r'\s*:\s*'</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Regex.split</code>:</p>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html b/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html
index a817fe2..ed383cc 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html
@@ -558,8 +558,7 @@ You can specify a different delimiter using the <code class="highlighter-rouge">
<span class="p">(</span><span class="s">'🥔'</span><span class="p">,</span> <span class="s">'Potato'</span><span class="p">),</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'To string'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ToString</span><span class="o">.</span><span class="n">Kvs</span><span class="p">()</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">ToString</code>:</p>
@@ -645,8 +644,7 @@ The string output will be equivalent to <code class="highlighter-rouge">str(elem
<span class="p">[</span><span class="s">'🥔'</span><span class="p">,</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'perennial'</span><span class="p">],</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'To string'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ToString</span><span class="o">.</span><span class="n">Element</span><span class="p">()</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">ToString</code>:</p>
@@ -734,8 +732,7 @@ The string output will be equivalent to <code class="highlighter-rouge">iterable
<span class="p">[</span><span class="s">'🥔'</span><span class="p">,</span> <span class="s">'Potato'</span><span class="p">,</span> <span class="s">'perennial'</span><span class="p">],</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'To string'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ToString</span><span class="o">.</span><span class="n">Iterables</span><span class="p">()</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">ToString</code>:</p>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/values/index.html b/website/generated-content/documentation/transforms/python/elementwise/values/index.html
index dd7f2a5..caee3e2 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/values/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/values/index.html
@@ -545,8 +545,7 @@ Then, we apply <code class="highlighter-rouge">Values</code> to extract the valu
<span class="p">(</span><span class="s">'🥔'</span><span class="p">,</span> <span class="s">'Potato'</span><span class="p">),</span>
<span class="p">])</span>
<span class="o">|</span> <span class="s">'Values'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Values</span><span class="p">()</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">)</span>
- <span class="p">)</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">print</span><span class="p">))</span>
</code></pre></div></div>
<p class="notebook-skip">Output <code class="highlighter-rouge">PCollection</code> after <code class="highlighter-rouge">Values</code>:</p>
diff --git a/website/generated-content/get-started/mobile-gaming-example/index.html b/website/generated-content/get-started/mobile-gaming-example/index.html
index 1468ef1..f4000aa 100644
--- a/website/generated-content/get-started/mobile-gaming-example/index.html
+++ b/website/generated-content/get-started/mobile-gaming-example/index.html
@@ -418,9 +418,12 @@ looks more like what is depicted by the red squiggly line above the ideal line.<
<span class="bp">self</span><span class="o">.</span><span class="n">field</span> <span class="o">=</span> <span class="n">field</span>
<span class="k">def</span> <span class="nf">expand</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pcoll</span><span class="p">):</span>
- <span class="k">return</span> <span class="p">(</span><span class="n">pcoll</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="p">(</span><span class="n">elem</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">field</span><span class="p">],</span> <span class="n">elem</span><span class="p">[</span><span class="s">'score'</span><span clas [...]
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombinePerKey</span><span class="p">(</span><span class="nb">sum</span><span class="p">))</span>
+ <span class="k">return</span> <span class="p">(</span>
+ <span class="n">pcoll</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="p">(</span><span class="n">elem</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">field</span><span class="p">],</span> <span class="n">elem</span><span class="p">[</span><span class="s">'score'</span><span class="p [...]
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombinePerKey</span><span class="p">(</span><span class="nb">sum</span><span class="p">))</span>
+
+
</code></pre></div></div>
<p><code class="highlighter-rouge">ExtractAndSumScore</code> is written to be more general, in that you can pass in the field by which you want to group the data (in the case of our game, by unique user or unique team). This means we can re-use <code class="highlighter-rouge">ExtractAndSumScore</code> in other pipelines that group score data by team, for example.</p>
@@ -451,14 +454,13 @@ looks more like what is depicted by the red squiggly line above the ideal line.<
<span class="c"># The default maps to two large Google Cloud Storage files (each ~12GB)</span>
<span class="c"># holding two subsequent day's worth (roughly) of data.</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--input'</span><span class="p">,</span>
- <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
- <span class="n">default</span><span class="o">=</span><span class="s">'gs://apache-beam-samples/game/gaming_data*.csv'</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'Path to the data file(s) containing game data.'</span><span class="p">)</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--output'</span><span class="p">,</span>
- <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
- <span class="n">required</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'Path to the output file(s).'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--input'</span><span class="p">,</span>
+ <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
+ <span class="n">default</span><span class="o">=</span><span class="s">'gs://apache-beam-samples/game/gaming_data*.csv'</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'Path to the data file(s) containing game data.'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--output'</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span> <span class="n">required</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">help</span><span class="o">=</span><span class="s">'Path to the output file(s).'</span><span class="p">)</span>
<span class="n">args</span><span class="p">,</span> <span class="n">pipeline_args</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_known_args</span><span class="p">(</span><span class="n">argv</span><span class="p">)</span>
@@ -469,15 +471,19 @@ looks more like what is depicted by the red squiggly line above the ideal line.<
<span class="n">options</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">SetupOptions</span><span class="p">)</span><span class="o">.</span><span class="n">save_main_session</span> <span class="o">=</span> <span class="n">save_main_session</span>
<span class="k">with</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">(</span><span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">)</span> <span class="k">as</span> <span class="n">p</span><span class="p">:</span>
+
<span class="k">def</span> <span class="nf">format_user_score_sums</span><span class="p">(</span><span class="n">user_score</span><span class="p">):</span>
<span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">score</span><span class="p">)</span> <span class="o">=</span> <span class="n">user_score</span>
<span class="k">return</span> <span class="s">'user: </span><span class="si">%</span><span class="s">s, total_score: </span><span class="si">%</span><span class="s">s'</span> <span class="o">%</span> <span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">score</span><span class="p">)</span>
- <span class="p">(</span><span class="n">p</span> <span class="c"># pylint: disable=expression-not-assigned</span>
- <span class="o">|</span> <span class="s">'ReadInputText'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="nb">input</span><span class="p">)</span>
- <span class="o">|</span> <span class="s">'UserScore'</span> <span class="o">>></span> <span class="n">UserScore</span><span class="p">()</span>
- <span class="o">|</span> <span class="s">'FormatUserScoreSums'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="n">format_user_score_sums</span><span class="p">)</span>
- <span class="o">|</span> <span class="s">'WriteUserScoreSums'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">WriteToText</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="n">output</span><span class="p">))</span>
+ <span class="p">(</span> <span class="c"># pylint: disable=expression-not-assigned</span>
+ <span class="n">p</span>
+ <span class="o">|</span> <span class="s">'ReadInputText'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="nb">input</span><span class="p">)</span>
+ <span class="o">|</span> <span class="s">'UserScore'</span> <span class="o">>></span> <span class="n">UserScore</span><span class="p">()</span>
+ <span class="o">|</span> <span class="s">'FormatUserScoreSums'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="n">format_user_score_sums</span><span class="p">)</span>
+ <span class="o">|</span> <span class="s">'WriteUserScoreSums'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">WriteToText</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="n">output</span><span class="p">))</span>
+
+
</code></pre></div></div>
<h3 id="limitations">Limitations</h3>
@@ -592,10 +598,10 @@ logical windows based on when those scores occurred in event time.</em></p>
<span class="n">Filter</span><span class="o">.</span><span class="na">by</span><span class="o">(</span>
<span class="o">(</span><span class="n">GameActionInfo</span> <span class="n">gInfo</span><span class="o">)</span> <span class="o">-></span> <span class="n">gInfo</span><span class="o">.</span><span class="na">getTimestamp</span><span class="o">()</span> <span class="o"><</span> <span class="n">stopMinTimestamp</span><span class="o">.</span><span class="na">getMillis</span><span class="o">()))</span>
</code></pre></div></div>
-<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="o">|</span> <span class="s">'FilterStartTime'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'timestamp'</span><span class="p">]</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">start_timestamp</span><span class="p">)</span>
-<span class="o">|</span> <span class="s">'FilterEndTime'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'timestamp'</span><span class="p">]</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">stop_timestamp</span><span class="p">)</span>
+<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="o">|</span> <span class="s">'FilterStartTime'</span> <span class="o">>></span>
+<span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'timestamp'</span><span class="p">]</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">start_timestamp</span><span class="p">)</span>
+<span class="o">|</span> <span class="s">'FilterEndTime'</span> <span class="o">>></span>
+<span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'timestamp'</span><span class="p">]</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">stop_timestamp</span><span class="p">)</span>
</code></pre></div></div>
<h4 id="calculating-score-per-team-per-window">Calculating Score Per Team, Per Window</h4>
@@ -675,10 +681,10 @@ logical windows based on when those scores occurred in event time.</em></p>
<span class="c"># need to weed out events that fall after the time period we want to</span>
<span class="c"># analyze.</span>
<span class="c"># [START filter_by_time_range]</span>
- <span class="o">|</span> <span class="s">'FilterStartTime'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'timestamp'</span><span class="p">]</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">start_timestamp</span><span class="p">)</span>
- <span class="o">|</span> <span class="s">'FilterEndTime'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'timestamp'</span><span class="p">]</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">stop_timestamp</span><span class="p">)</span>
+ <span class="o">|</span> <span class="s">'FilterStartTime'</span> <span class="o">>></span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'timestamp'</span><span class="p">]</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">start_timestamp</span><span class="p">)</span>
+ <span class="o">|</span> <span class="s">'FilterEndTime'</span> <span class="o">>></span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'timestamp'</span><span class="p">]</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">stop_timestamp</span><span class="p">)</span>
<span class="c"># [END filter_by_time_range]</span>
<span class="c"># [START add_timestamp_and_window]</span>
@@ -700,38 +706,44 @@ logical windows based on when those scores occurred in event time.</em></p>
<span class="c"># The default maps to two large Google Cloud Storage files (each ~12GB)</span>
<span class="c"># holding two subsequent day's worth (roughly) of data.</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--input'</span><span class="p">,</span>
- <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
- <span class="n">default</span><span class="o">=</span><span class="s">'gs://apache-beam-samples/game/gaming_data*.csv'</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'Path to the data file(s) containing game data.'</span><span class="p">)</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--dataset'</span><span class="p">,</span>
- <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
- <span class="n">required</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'BigQuery Dataset to write tables to. '</span>
- <span class="s">'Must already exist.'</span><span class="p">)</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--table_name'</span><span class="p">,</span>
- <span class="n">default</span><span class="o">=</span><span class="s">'leader_board'</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'The BigQuery table name. Should not already exist.'</span><span class="p">)</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--window_duration'</span><span class="p">,</span>
- <span class="nb">type</span><span class="o">=</span><span class="nb">int</span><span class="p">,</span>
- <span class="n">default</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'Numeric value of fixed window duration, in minutes'</span><span class="p">)</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--start_min'</span><span class="p">,</span>
- <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
- <span class="n">default</span><span class="o">=</span><span class="s">'1970-01-01-00-00'</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'String representation of the first minute after '</span>
- <span class="s">'which to generate results in the format: '</span>
- <span class="s">'yyyy-MM-dd-HH-mm. Any input data timestamped '</span>
- <span class="s">'prior to that minute won</span><span class="se">\'</span><span class="s">t be included in the '</span>
- <span class="s">'sums.'</span><span class="p">)</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--stop_min'</span><span class="p">,</span>
- <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
- <span class="n">default</span><span class="o">=</span><span class="s">'2100-01-01-00-00'</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'String representation of the first minute for '</span>
- <span class="s">'which to generate results in the format: '</span>
- <span class="s">'yyyy-MM-dd-HH-mm. Any input data timestamped '</span>
- <span class="s">'after to that minute won</span><span class="se">\'</span><span class="s">t be included in the '</span>
- <span class="s">'sums.'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--input'</span><span class="p">,</span>
+ <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
+ <span class="n">default</span><span class="o">=</span><span class="s">'gs://apache-beam-samples/game/gaming_data*.csv'</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'Path to the data file(s) containing game data.'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--dataset'</span><span class="p">,</span>
+ <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
+ <span class="n">required</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'BigQuery Dataset to write tables to. '</span>
+ <span class="s">'Must already exist.'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--table_name'</span><span class="p">,</span>
+ <span class="n">default</span><span class="o">=</span><span class="s">'leader_board'</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'The BigQuery table name. Should not already exist.'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--window_duration'</span><span class="p">,</span>
+ <span class="nb">type</span><span class="o">=</span><span class="nb">int</span><span class="p">,</span>
+ <span class="n">default</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'Numeric value of fixed window duration, in minutes'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--start_min'</span><span class="p">,</span>
+ <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
+ <span class="n">default</span><span class="o">=</span><span class="s">'1970-01-01-00-00'</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'String representation of the first minute after '</span>
+ <span class="s">'which to generate results in the format: '</span>
+ <span class="s">'yyyy-MM-dd-HH-mm. Any input data timestamped '</span>
+ <span class="s">'prior to that minute won</span><span class="se">\'</span><span class="s">t be included in the '</span>
+ <span class="s">'sums.'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--stop_min'</span><span class="p">,</span>
+ <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
+ <span class="n">default</span><span class="o">=</span><span class="s">'2100-01-01-00-00'</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'String representation of the first minute for '</span>
+ <span class="s">'which to generate results in the format: '</span>
+ <span class="s">'yyyy-MM-dd-HH-mm. Any input data timestamped '</span>
+ <span class="s">'after to that minute won</span><span class="se">\'</span><span class="s">t be included in the '</span>
+ <span class="s">'sums.'</span><span class="p">)</span>
<span class="n">args</span><span class="p">,</span> <span class="n">pipeline_args</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_known_args</span><span class="p">(</span><span class="n">argv</span><span class="p">)</span>
@@ -748,17 +760,23 @@ logical windows based on when those scores occurred in event time.</em></p>
<span class="n">options</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">SetupOptions</span><span class="p">)</span><span class="o">.</span><span class="n">save_main_session</span> <span class="o">=</span> <span class="n">save_main_session</span>
<span class="k">with</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">(</span><span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">)</span> <span class="k">as</span> <span class="n">p</span><span class="p">:</span>
- <span class="p">(</span><span class="n">p</span> <span class="c"># pylint: disable=expression-not-assigned</span>
- <span class="o">|</span> <span class="s">'ReadInputText'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="nb">input</span><span class="p">)</span>
- <span class="o">|</span> <span class="s">'HourlyTeamScore'</span> <span class="o">>></span> <span class="n">HourlyTeamScore</span><span class="p">(</span>
- <span class="n">args</span><span class="o">.</span><span class="n">start_min</span><span class="p">,</span> <span class="n">args</span><span class="o">.</span><span class="n">stop_min</span><span class="p">,</span> <span class="n">args</span><span class="o">.</span><span class="n">window_duration</span><span class="p">)</span>
- <span class="o">|</span> <span class="s">'TeamScoresDict'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">TeamScoresDict</span><span class="p">())</span>
- <span class="o">|</span> <span class="s">'WriteTeamScoreSums'</span> <span class="o">>></span> <span class="n">WriteToBigQuery</span><span class="p">(</span>
- <span class="n">args</span><span class="o">.</span><span class="n">table_name</span><span class="p">,</span> <span class="n">args</span><span class="o">.</span><span class="n">dataset</span><span class="p">,</span> <span class="p">{</span>
- <span class="s">'team'</span><span class="p">:</span> <span class="s">'STRING'</span><span class="p">,</span>
- <span class="s">'total_score'</span><span class="p">:</span> <span class="s">'INTEGER'</span><span class="p">,</span>
- <span class="s">'window_start'</span><span class="p">:</span> <span class="s">'STRING'</span><span class="p">,</span>
- <span class="p">},</span> <span class="n">options</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">GoogleCloudOptions</span><span class="p">)</span><span class="o">.</span><span class="n">project</span><span class="p">))</span>
+ <span class="p">(</span> <span class="c"># pylint: disable=expression-not-assigned</span>
+ <span class="n">p</span>
+ <span class="o">|</span> <span class="s">'ReadInputText'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="nb">input</span><span class="p">)</span>
+ <span class="o">|</span> <span class="s">'HourlyTeamScore'</span> <span class="o">>></span> <span class="n">HourlyTeamScore</span><span class="p">(</span>
+ <span class="n">args</span><span class="o">.</span><span class="n">start_min</span><span class="p">,</span> <span class="n">args</span><span class="o">.</span><span class="n">stop_min</span><span class="p">,</span> <span class="n">args</span><span class="o">.</span><span class="n">window_duration</span><span class="p">)</span>
+ <span class="o">|</span> <span class="s">'TeamScoresDict'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">TeamScoresDict</span><span class="p">())</span>
+ <span class="o">|</span> <span class="s">'WriteTeamScoreSums'</span> <span class="o">>></span> <span class="n">WriteToBigQuery</span><span class="p">(</span>
+ <span class="n">args</span><span class="o">.</span><span class="n">table_name</span><span class="p">,</span>
+ <span class="n">args</span><span class="o">.</span><span class="n">dataset</span><span class="p">,</span>
+ <span class="p">{</span>
+ <span class="s">'team'</span><span class="p">:</span> <span class="s">'STRING'</span><span class="p">,</span>
+ <span class="s">'total_score'</span><span class="p">:</span> <span class="s">'INTEGER'</span><span class="p">,</span>
+ <span class="s">'window_start'</span><span class="p">:</span> <span class="s">'STRING'</span><span class="p">,</span>
+ <span class="p">},</span>
+ <span class="n">options</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">GoogleCloudOptions</span><span class="p">)</span><span class="o">.</span><span class="n">project</span><span class="p">))</span>
+
+
</code></pre></div></div>
<h3 id="limitations-1">Limitations</h3>
@@ -873,6 +891,8 @@ ten minutes after data is received.</em></p>
<span class="n">accumulation_mode</span><span class="o">=</span><span class="n">trigger</span><span class="o">.</span><span class="n">AccumulationMode</span><span class="o">.</span><span class="n">ACCUMULATING</span><span class="p">)</span>
<span class="c"># Extract and sum username/score pairs from the event data.</span>
<span class="o">|</span> <span class="s">'ExtractAndSumScore'</span> <span class="o">>></span> <span class="n">ExtractAndSumScore</span><span class="p">(</span><span class="s">'user'</span><span class="p">))</span>
+
+
</code></pre></div></div>
<p><code class="highlighter-rouge">LeaderBoard</code> sets the <a href="/documentation/programming-guide/#window-accumulation-modes">window accumulation mode</a> to accumulate window panes as the trigger fires. This accumulation mode is set by <span class="language-java">invoking <code class="highlighter-rouge">.accumulatingFiredPanes</code></span> <span class="language-py">using <code class="highlighter-rouge">accumulation_mode=trigger.AccumulationMode.ACCUMULATING</code></span> when se [...]
@@ -959,11 +979,13 @@ late results.</em></p>
<span class="c"># processing of late data.</span>
<span class="o">|</span> <span class="s">'LeaderboardTeamFixedWindows'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">WindowInto</span><span class="p">(</span>
<span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">FixedWindows</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">team_window_duration</span><span class="p">),</span>
- <span class="n">trigger</span><span class="o">=</span><span class="n">trigger</span><span class="o">.</span><span class="n">AfterWatermark</span><span class="p">(</span><span class="n">trigger</span><span class="o">.</span><span class="n">AfterCount</span><span class="p">(</span><span class="mi">10</span><span class="p">),</span>
- <span class="n">trigger</span><span class="o">.</span><span class="n">AfterCount</span><span class="p">(</span><span class="mi">20</span><span class="p">)),</span>
+ <span class="n">trigger</span><span class="o">=</span><span class="n">trigger</span><span class="o">.</span><span class="n">AfterWatermark</span><span class="p">(</span>
+ <span class="n">trigger</span><span class="o">.</span><span class="n">AfterCount</span><span class="p">(</span><span class="mi">10</span><span class="p">),</span> <span class="n">trigger</span><span class="o">.</span><span class="n">AfterCount</span><span class="p">(</span><span class="mi">20</span><span class="p">)),</span>
<span class="n">accumulation_mode</span><span class="o">=</span><span class="n">trigger</span><span class="o">.</span><span class="n">AccumulationMode</span><span class="o">.</span><span class="n">ACCUMULATING</span><span class="p">)</span>
<span class="c"># Extract and sum teamname/score pairs from the event data.</span>
<span class="o">|</span> <span class="s">'ExtractAndSumScore'</span> <span class="o">>></span> <span class="n">ExtractAndSumScore</span><span class="p">(</span><span class="s">'team'</span><span class="p">))</span>
+
+
</code></pre></div></div>
<p>Taken together, these processing strategies let us address the latency and completeness issues present in the <code class="highlighter-rouge">UserScore</code> and <code class="highlighter-rouge">HourlyTeamScore</code> pipelines, while still using the same basic transforms to process the data—as a matter of fact, both calculations still use the same <code class="highlighter-rouge">ExtractAndSumScore</code> transform that we used in both the <code class="highlighter-rouge">UserScore</co [...]
@@ -1064,9 +1086,7 @@ late results.</em></p>
<span class="k">def</span> <span class="nf">expand</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">user_scores</span><span class="p">):</span>
<span class="c"># Get the sum of scores for each user.</span>
- <span class="n">sum_scores</span> <span class="o">=</span> <span class="p">(</span>
- <span class="n">user_scores</span>
- <span class="o">|</span> <span class="s">'SumUsersScores'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">CombinePerKey</span><span class="p">(</span><span class="nb">sum</span><span class="p">))</span>
+ <span class="n">sum_scores</span> <span class="o">=</span> <span class="p">(</span><span class="n">user_scores</span> <span class="o">|</span> <span class="s">'SumUsersScores'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">CombinePerKey</span><span class="p">(</span><span class="nb">sum</span><span class="p">))</span>
<span class="c"># Extract the score from each element, and use it to find the global mean.</span>
<span class="n">global_mean_score</span> <span class="o">=</span> <span class="p">(</span>
@@ -1084,6 +1104,8 @@ late results.</em></p>
<span class="n">key_score</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">></span> <span class="n">global_mean</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">SCORE_WEIGHT</span><span class="p">,</span>
<span class="n">global_mean_score</span><span class="p">))</span>
<span class="k">return</span> <span class="n">filtered</span>
+
+
</code></pre></div></div>
<p>The abuse-detection transform generates a view of users supected to be spambots. Later in the pipeline, we use that view to filter out any such users when we calculate the team score per hour, again by using the side input mechanism. The following code example shows where we insert the spam filter, between windowing the scores into fixed windows and extracting the team scores:</p>
@@ -1118,16 +1140,17 @@ late results.</em></p>
<span class="c"># updates for late data. Uses the side input derived above --the set of</span>
<span class="c"># suspected robots-- to filter out scores from those users from the sum.</span>
<span class="c"># Write the results to BigQuery.</span>
-<span class="p">(</span><span class="n">raw_events</span> <span class="c"># pylint: disable=expression-not-assigned</span>
- <span class="o">|</span> <span class="s">'WindowIntoFixedWindows'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">WindowInto</span><span class="p">(</span>
- <span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">FixedWindows</span><span class="p">(</span><span class="n">fixed_window_duration</span><span class="p">))</span>
-
- <span class="c"># Filter out the detected spammer users, using the side input derived above</span>
- <span class="o">|</span> <span class="s">'FilterOutSpammers'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">elem</span><span class="p">,</span> <span class="n">spammers</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'user'</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">spammers</span><span class="p">,</span>
- <span class="n">spammers_view</span><span class="p">)</span>
- <span class="c"># Extract and sum teamname/score pairs from the event data.</span>
- <span class="o">|</span> <span class="s">'ExtractAndSumScore'</span> <span class="o">>></span> <span class="n">ExtractAndSumScore</span><span class="p">(</span><span class="s">'team'</span><span class="p">)</span>
+<span class="p">(</span> <span class="c"># pylint: disable=expression-not-assigned</span>
+ <span class="n">raw_events</span>
+ <span class="o">|</span> <span class="s">'WindowIntoFixedWindows'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">WindowInto</span><span class="p">(</span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">FixedWindows</span><span class="p">(</span><span class="n">fixed_window_duration</span><span class="p">))</span>
+
+ <span class="c"># Filter out the detected spammer users, using the side input derived</span>
+ <span class="c"># above</span>
+ <span class="o">|</span> <span class="s">'FilterOutSpammers'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Filter</span><span class="p">(</span>
+ <span class="k">lambda</span> <span class="n">elem</span><span class="p">,</span> <span class="n">spammers</span><span class="p">:</span> <span class="n">elem</span><span class="p">[</span><span class="s">'user'</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">spammers</span><span class="p">,</span> <span class="n">spammers_view</span><span class="p">)</span>
+ <span class="c"># Extract and sum teamname/score pairs from the event data.</span>
+ <span class="o">|</span> <span class="s">'ExtractAndSumScore'</span> <span class="o">>></span> <span class="n">ExtractAndSumScore</span><span class="p">(</span><span class="s">'team'</span><span class="p">)</span>
</code></pre></div></div>
<h4 id="analyzing-usage-patterns">Analyzing Usage Patterns</h4>
@@ -1166,18 +1189,19 @@ between instances are.</em></p>
<span class="c"># from further activity. Find and record the mean session lengths.</span>
<span class="c"># This information could help the game designers track the changing user</span>
<span class="c"># engagement as their set of game changes.</span>
-<span class="p">(</span><span class="n">user_events</span> <span class="c"># pylint: disable=expression-not-assigned</span>
- <span class="o">|</span> <span class="s">'WindowIntoSessions'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">WindowInto</span><span class="p">(</span>
- <span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">Sessions</span><span class="p">(</span><span class="n">session_gap</span><span class="p">),</span>
- <span class="n">timestamp_combiner</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">TimestampCombiner</span><span class="o">.</span><span class="n">OUTPUT_AT_EOW</span><span class="p">)</span>
-
- <span class="c"># For this use, we care only about the existence of the session, not any</span>
- <span class="c"># particular information aggregated over it, so we can just group by key</span>
- <span class="c"># and assign a "dummy value" of None.</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombinePerKey</span><span class="p">(</span><span class="k">lambda</span> <span class="n">_</span><span class="p">:</span> <span class="bp">None</span><span class="p">)</span>
-
- <span class="c"># Get the duration of the session</span>
- <span class="o">|</span> <span class="s">'UserSessionActivity'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">UserSessionActivity</span><span class="p">())</span>
+<span class="p">(</span> <span class="c"># pylint: disable=expression-not-assigned</span>
+ <span class="n">user_events</span>
+ <span class="o">|</span> <span class="s">'WindowIntoSessions'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">WindowInto</span><span class="p">(</span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">Sessions</span><span class="p">(</span><span class="n">session_gap</span><span class="p">),</span>
+ <span class="n">timestamp_combiner</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">TimestampCombiner</span><span class="o">.</span><span class="n">OUTPUT_AT_EOW</span><span class="p">)</span>
+
+ <span class="c"># For this use, we care only about the existence of the session, not any</span>
+ <span class="c"># particular information aggregated over it, so we can just group by key</span>
+ <span class="c"># and assign a "dummy value" of None.</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombinePerKey</span><span class="p">(</span><span class="k">lambda</span> <span class="n">_</span><span class="p">:</span> <span class="bp">None</span><span class="p">)</span>
+
+ <span class="c"># Get the duration of the session</span>
+ <span class="o">|</span> <span class="s">'UserSessionActivity'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span><span class="n">UserSessionActivity</span><span class="p">())</span>
</code></pre></div></div>
<p>This gives us a set of user sessions, each with an attached duration. We can then calculate the <em>average</em> session length by re-windowing the data into fixed time windows, and then calculating the average for all sessions that end in each hour:</p>
@@ -1204,13 +1228,16 @@ between instances are.</em></p>
<span class="n">beam</span><span class="o">.</span><span class="n">window</span><span class="o">.</span><span class="n">FixedWindows</span><span class="p">(</span><span class="n">user_activity_window_duration</span><span class="p">))</span>
<span class="c"># Find the mean session duration in each window</span>
-<span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombineGlobally</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">MeanCombineFn</span><span class="p">())</span><span class="o">.</span><span class="n">without_defaults</span><span class="p">()</span>
-<span class="o">|</span> <span class="s">'FormatAvgSessionLength'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="p">{</span><span class="s">'mean_duration'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">elem</span><span class="p">)})</span>
+<span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">CombineGlobally</span><span class="p">(</span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">MeanCombineFn</span><span class="p">())</span><span class="o">.</span><span class="n">without_defaults</span><span class="p">()</span>
+<span class="o">|</span> <span class="s">'FormatAvgSessionLength'</span> <span class="o">>></span>
+<span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">elem</span><span class="p">:</span> <span class="p">{</span><span class="s">'mean_duration'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">elem</span><span class="p">)})</span>
<span class="o">|</span> <span class="s">'WriteAvgSessionLength'</span> <span class="o">>></span> <span class="n">WriteToBigQuery</span><span class="p">(</span>
- <span class="n">args</span><span class="o">.</span><span class="n">table_name</span> <span class="o">+</span> <span class="s">'_sessions'</span><span class="p">,</span> <span class="n">args</span><span class="o">.</span><span class="n">dataset</span><span class="p">,</span> <span class="p">{</span>
+ <span class="n">args</span><span class="o">.</span><span class="n">table_name</span> <span class="o">+</span> <span class="s">'_sessions'</span><span class="p">,</span>
+ <span class="n">args</span><span class="o">.</span><span class="n">dataset</span><span class="p">,</span> <span class="p">{</span>
<span class="s">'mean_duration'</span><span class="p">:</span> <span class="s">'FLOAT'</span><span class="p">,</span>
- <span class="p">},</span> <span class="n">options</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">GoogleCloudOptions</span><span class="p">)</span><span class="o">.</span><span class="n">project</span><span class="p">))</span>
+ <span class="p">},</span>
+ <span class="n">options</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">GoogleCloudOptions</span><span class="p">)</span><span class="o">.</span><span class="n">project</span><span class="p">))</span>
</code></pre></div></div>
<p>We can use the resulting information to find, for example, what times of day our users are playing the longest, or which stretches of the day are more likely to see shorter play sessions.</p>
diff --git a/website/generated-content/get-started/wordcount-example/index.html b/website/generated-content/get-started/wordcount-example/index.html
index 7db7893..8fef84f 100644
--- a/website/generated-content/get-started/wordcount-example/index.html
+++ b/website/generated-content/get-started/wordcount-example/index.html
@@ -461,8 +461,8 @@ data stored in a publicly accessible Google Cloud Storage bucket (“gs://”).<
<div class="language-java highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">TextIO</span><span class="o">.</span><span class="na">read</span><span class="o">().</span><span class="na">from</span><span class="o">(</span><span class="s">"gs://apache-beam-samples/shakespeare/*"</span><span class="o">))</span>
</code></pre></div> </div>
- <div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">p</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">(</span>
-<span class="s">'gs://dataflow-samples/shakespeare/kinglear.txt'</span><span class="p">)</span>
+ <div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">p</span>
+<span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">(</span><span class="s">'gs://dataflow-samples/shakespeare/kinglear.txt'</span><span class="p">)</span>
</code></pre></div> </div>
<div class="language-go highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">lines</span><span class="x"> </span><span class="o">:=</span><span class="x"> </span><span class="n">textio</span><span class="o">.</span><span class="n">Read</span><span class="p">(</span><span class="n">s</span><span class="p">,</span><span class="x"> </span><span class="s">"gs://apache-beam-samples/shakespeare/*"</span><span class="p">)</span><span class="x">
@@ -764,7 +764,6 @@ test and can make the <code class="highlighter-rouge">ParDo</code> code more rea
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># In this example, the DoFns are defined as classes:</span>
<span class="k">class</span> <span class="nc">FormatAsTextFn</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="p">):</span>
-
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">element</span><span class="p">):</span>
<span class="n">word</span><span class="p">,</span> <span class="n">count</span> <span class="o">=</span> <span class="n">element</span>
<span class="k">yield</span> <span class="s">'</span><span class="si">%</span><span class="s">s: </span><span class="si">%</span><span class="s">s'</span> <span class="o">%</span> <span class="p">(</span><span class="n">word</span><span class="p">,</span> <span class="n">count</span><span class="p">)</span>
@@ -828,17 +827,18 @@ is the <code class="highlighter-rouge">PCollection<KV<String, Long>>
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">class</span> <span class="nc">CountWords</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">PTransform</span><span class="p">):</span>
-
<span class="k">def</span> <span class="nf">expand</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pcoll</span><span class="p">):</span>
- <span class="k">return</span> <span class="p">(</span><span class="n">pcoll</span>
- <span class="c"># Convert lines of text into individual words.</span>
- <span class="o">|</span> <span class="s">'ExtractWords'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span>
- <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="s">r'[A-Za-z\']+'</span><span class="p">,</span> <span class="n">x</span><span class="p">))</span>
+ <span class="k">return</span> <span class="p">(</span>
+ <span class="n">pcoll</span>
+ <span class="c"># Convert lines of text into individual words.</span>
+ <span class="o">|</span> <span class="s">'ExtractWords'</span> <span class="o">>></span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="s">r'[A-Za-z\']+'</span><span class="p">,</span> <span class="n">x</span><span class="p">))</span>
- <span class="c"># Count the number of times each word occurs.</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">Count</span><span class="o">.</span><span class="n">PerElement</span><span class="p">())</span>
+ <span class="c"># Count the number of times each word occurs.</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">Count</span><span class="o">.</span><span class="n">PerElement</span><span class="p">())</span>
<span class="n">counts</span> <span class="o">=</span> <span class="n">lines</span> <span class="o">|</span> <span class="n">CountWords</span><span class="p">()</span>
+
</code></pre></div></div>
<div class="language-go highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">func</span><span class="x"> </span><span class="n">CountWords</span><span class="p">(</span><span class="n">s</span><span class="x"> </span><span class="n">beam</span><span class="o">.</span><span class="n">Scope</span><span class="p">,</span><span class="x"> </span><span class="n">lines</span><span class="x"> </span><span class="n">beam</span><span class="o">.</span><span class [...]
@@ -881,17 +881,18 @@ values for them. You can then access the options values in your pipeline code.</
</code></pre></div></div>
<div class="language-py highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">class</span> <span class="nc">WordCountOptions</span><span class="p">(</span><span class="n">PipelineOptions</span><span class="p">):</span>
-
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">_add_argparse_args</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">parser</span><span class="p">):</span>
- <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s">'--input'</span><span class="p">,</span>
- <span class="n">help</span><span class="o">=</span><span class="s">'Input for the pipeline'</span><span class="p">,</span>
- <span class="n">default</span><span class="o">=</span><span class="s">'gs://my-bucket/input'</span><span class="p">)</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
+ <span class="s">'--input'</span><span class="p">,</span>
+ <span class="n">help</span><span class="o">=</span><span class="s">'Input for the pipeline'</span><span class="p">,</span>
+ <span class="n">default</span><span class="o">=</span><span class="s">'gs://my-bucket/input'</span><span class="p">)</span>
<span class="n">options</span> <span class="o">=</span> <span class="n">PipelineOptions</span><span class="p">(</span><span class="n">argv</span><span class="p">)</span>
<span class="n">word_count_options</span> <span class="o">=</span> <span class="n">options</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">WordCountOptions</span><span class="p">)</span>
<span class="k">with</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">(</span><span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">)</span> <span class="k">as</span> <span class="n">p</span><span class="p">:</span>
<span class="n">lines</span> <span class="o">=</span> <span class="n">p</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">ReadFromText</span><span class="p">(</span><span class="n">word_count_options</span><span class="o">.</span><span class="nb">input</span><span class="p">)</span>
+
</code></pre></div></div>
<div class="language-go highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">var</span><span class="x"> </span><span class="n">input</span><span class="x"> </span><span class="o">=</span><span class="x"> </span><span class="n">flag</span><span class="o">.</span><span class="n">String</span><span class="p">(</span><span class="s">"input"</span><span class="p">,</span><span class="x"> </span><span class="s">"gs://apache-beam-samples/shakespeare/kinglear.tx [...]
@@ -1076,7 +1077,6 @@ pipeline code into smaller sections.</p>
<span class="k">class</span> <span class="nc">FilterTextFn</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="p">):</span>
<span class="s">"""A DoFn that filters for a specific key based on a regular expression."""</span>
-
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pattern</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pattern</span> <span class="o">=</span> <span class="n">pattern</span>
<span class="c"># A custom metric can track values in your pipeline as it runs. Create</span>
@@ -1105,6 +1105,7 @@ pipeline code into smaller sections.</p>
<span class="c"># Add 1 to the custom metric counter umatched_words</span>
<span class="bp">self</span><span class="o">.</span><span class="n">umatched_words</span><span class="o">.</span><span class="n">inc</span><span class="p">()</span>
+
</code></pre></div></div>
<div class="language-go highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">type</span><span class="x"> </span><span class="n">filterFn</span><span class="x"> </span><span class="k">struct</span><span class="x"> </span><span class="p">{</span><span class="x">