You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/08/28 23:01:43 UTC
[beam] branch asf-site updated: Publishing website 2020/08/28
23:01:29 at commit 42b1911
This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/asf-site by this push:
new b99f82d Publishing website 2020/08/28 23:01:29 at commit 42b1911
b99f82d is described below
commit b99f82d7d7d8c6cc49fb711b89c22563244dd5b2
Author: jenkins <us...@infra.apache.org>
AuthorDate: Fri Aug 28 23:01:29 2020 +0000
Publishing website 2020/08/28 23:01:29 at commit 42b1911
---
.../generated-content/blog/beam-kotlin/index.html | 2 +-
website/generated-content/blog/index.html | 6 +-
website/generated-content/blog/index.xml | 336 +++++++++++-----
.../blog/python-improved-annotations/index.html | 45 +++
.../index.html | 57 +++
.../blog/splittable-do-fn/index.html | 4 +-
.../blog/stateful-processing/index.html | 8 +-
.../blog/timely-processing/index.html | 4 +-
.../generated-content/categories/blog/index.xml | 244 +++++++++++-
website/generated-content/categories/index.xml | 2 +-
.../generated-content/categories/python/index.xml | 242 +++++++++++-
.../generated-content/categories/typing/index.xml | 242 +++++++++++-
.../contribute/release-guide/index.html | 34 +-
.../extensions/user-defined-functions/index.html | 12 +-
.../extensions/windowing-and-triggering/index.html | 20 +-
.../documentation/dsls/sql/shell/index.html | 2 +-
.../documentation/dsls/sql/walkthrough/index.html | 24 +-
website/generated-content/documentation/index.xml | 22 +-
.../documentation/io/built-in/hadoop/index.html | 2 +-
.../documentation/io/built-in/hcatalog/index.html | 2 +-
.../documentation/io/built-in/snowflake/index.html | 16 +-
.../documentation/io/testing/index.html | 2 +-
.../documentation/programming-guide/index.html | 174 +++++----
.../documentation/runners/jet/index.html | 4 +-
.../documentation/runners/samza/index.html | 3 +-
.../documentation/sdks/java-thirdparty/index.html | 2 +-
.../sdks/java/testing/nexmark/index.html | 2 +-
.../sdks/python-type-safety/index.html | 5 +-
website/generated-content/feed.xml | 429 ++++++++++++---------
website/generated-content/get-started/index.xml | 3 +-
.../get-started/quickstart-java/index.html | 29 +-
.../get-started/wordcount-example/index.html | 2 +-
website/generated-content/index.html | 2 +-
website/generated-content/security/index.html | 9 +-
website/generated-content/sitemap.xml | 2 +-
35 files changed, 1511 insertions(+), 483 deletions(-)
diff --git a/website/generated-content/blog/beam-kotlin/index.html b/website/generated-content/blog/beam-kotlin/index.html
index d93fdaf..82ccaa7 100644
--- a/website/generated-content/blog/beam-kotlin/index.html
+++ b/website/generated-content/blog/beam-kotlin/index.html
@@ -9,7 +9,7 @@ Harshit Dwivedi [<a href=https://twitter.com/harshithdwivedi>@harshithdwivedi</a
<span class=n>shardNumber</span><span class=o>,</span>
<span class=n>numShards</span><span class=o>,</span>
<span class=n>outputFileHints</span><span class=o>.</span><span class=na>suggestedFilenameSuffix</span><span class=o>);</span></code></pre></div></div><h3 id=kotlin>Kotlin</h3><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java> <span class=c1>// String templating
-</span><span class=c1></span> <span class=n>val</span> <span class=n>filename</span> <span class=o>=</span> <span class=s>"$filenamePrefixForWindow(intervalWindow)-$shardNumber-of-$numShards${outputFileHints.suggestedFilenameSuffix)"</span> </code></pre></div></div><h3 id=java-1>Java</h3><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=kd>public</span> <span class=kd>static</span> <span class=kd>class</span> <sp [...]
+</span><span class=c1></span> <span class=n>val</span> <span class=n>filename</span> <span class=o>=</span> <span class=s>"$filenamePrefixForWindow(intervalWindow)-$shardNumber-of-$numShards${outputFileHints.suggestedFilenameSuffix)"</span></code></pre></div></div><h3 id=java-1>Java</h3><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=kd>public</span> <span class=kd>static</span> <span class=kd>class</span> <span [...]
<span class=nd>@Override</span>
<span class=kd>public</span> <span class=n>String</span> <span class=nf>apply</span><span class=o>(</span><span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>Long</span><span class=o>></span> <span class=n>input</span><span class=o>)</span> <span class=o>{</span>
<span class=k>return</span> <span class=n>input</span><span class=o>.</span><span class=na>getKey</span><span class=o>()</span> <span class=o>+</span> <span class=s>": "</span> <span class=o>+</span> <span class=n>input</span><span class=o>.</span><span class=na>getValue</span><span class=o>();</span>
diff --git a/website/generated-content/blog/index.html b/website/generated-content/blog/index.html
index 0355884..f43817f 100644
--- a/website/generated-content/blog/index.html
+++ b/website/generated-content/blog/index.html
@@ -1,7 +1,11 @@
<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><title>Blogs</title><meta name=description content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languag [...]
<span class=sr-only>Toggle navigation</span>
<span class=icon-bar></span><span class=icon-bar></span><span class=icon-bar></span></button>
-<a href=/ class=navbar-brand><img alt=Brand style=height:25px src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask closed"></div><div id=navbar class="navbar-container closed"><ul class="nav navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a href=/documentation/>Documentation</a></li><li><a href=/documentation/sdks/java/>Languages</a></li><li><a href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a href=/roadmap/>Roadmap</a></li>< [...]
+<a href=/ class=navbar-brand><img alt=Brand style=height:25px src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask closed"></div><div id=navbar class="navbar-container closed"><ul class="nav navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a href=/documentation/>Documentation</a></li><li><a href=/documentation/sdks/java/>Languages</a></li><li><a href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a href=/roadmap/>Roadmap</a></li>< [...]
+•
+Saavan Nanavati</i></p>The importance of static type checking in a dynamically typed language like Python is not up for debate.<p><a class="btn btn-default btn-sm" href=/blog/python-improved-annotations/ role=button>Read more <span class="glyphicon glyphicon-menu-right" aria-hidden=true></span></a></p><hr><h3><a class=post-link href=/blog/python-performance-runtime-type-checking/>Performance-Driven Runtime Type Checking for the Python SDK</a></h3><p><i>Aug 21, 2020
+•
+Saavan Nanavati</i></p>In this blog post, we’re announcing the upcoming release of a new, opt-in runtime type checking system for Beam’s Python SDK that’s optimized for performance in both development and production environments.<p><a class="btn btn-default btn-sm" href=/blog/python-performance-runtime-type-checking/ role=button>Read more <span class="glyphicon glyphicon-menu-right" aria-hidden=true></span></a></p><hr><h3><a class=post-link href=/blog/beam-2.23.0/>Apache Beam 2.23.0 [...]
•
Valentyn Tymofieiev</i></p>We are happy to present the new 2.<p><a class="btn btn-default btn-sm" href=/blog/beam-2.23.0/ role=button>Read more <span class="glyphicon glyphicon-menu-right" aria-hidden=true></span></a></p><hr><h3><a class=post-link href=/blog/beam-2.22.0/>Apache Beam 2.22.0</a></h3><p><i>Jun 8, 2020
•
diff --git a/website/generated-content/blog/index.xml b/website/generated-content/blog/index.xml
index 31a6105..89c39f1 100644
--- a/website/generated-content/blog/index.xml
+++ b/website/generated-content/blog/index.xml
@@ -1,4 +1,244 @@
-<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – Blogs</title><link>/blog/</link><description>Recent content in Blogs on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Wed, 29 Jul 2020 00:00:01 -0800</lastBuildDate><atom:link href="/blog/index.xml" rel="self" type="application/rss+xml"/><item><title>Blog: Apache Beam 2.23.0</title><link>/blog/beam-2.23.0/</link><pubDate>Wed, 29 Jul 2020 00:00:01 -0800</pubDate><gui [...]
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – Blogs</title><link>/blog/</link><description>Recent content in Blogs on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Fri, 21 Aug 2020 00:00:01 -0800</lastBuildDate><atom:link href="/blog/index.xml" rel="self" type="application/rss+xml"/><item><title>Blog: Improved Annotation Support for the Python SDK</title><link>/blog/python-improved-annotations/</link><pubDate>F [...]
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>The importance of static type checking in a dynamically
+typed language like Python is not up for debate. Type hints
+allow developers to leverage a strong typing system to:</p>
+<ul>
+<li>write better code,</li>
+<li>self-document ambiguous programming logic, and</li>
+<li>inform intelligent code completion in IDEs like PyCharm.</li>
+</ul>
+<p>This is why we&rsquo;re excited to announce upcoming improvements to
+the <code>typehints</code> module of Beam&rsquo;s Python SDK, including support
+for typed PCollections and Python 3 style annotations on PTransforms.</p>
+<h1 id="improved-annotations">Improved Annotations</h1>
+<p>Today, you have the option to declare type hints on PTransforms using either
+class decorators or inline functions.</p>
+<p>For instance, a PTransform with decorated type hints might look like this:</p>
+<pre><code>@beam.typehints.with_input_types(int)
+@beam.typehints.with_output_types(str)
+class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>Using inline functions instead, the same transform would look like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr()).with_input_types(int).with_output_types(str)
+</code></pre><p>Both methods have problems. Class decorators are syntax-heavy,
+requiring two additional lines of code, whereas inline functions provide type hints
+that aren&rsquo;t reusable across other instances of the same transform. Additionally, both
+methods are incompatible with static type checkers like MyPy.</p>
+<p>With Python 3 annotations however, we can subvert these problems to provide a
+clean and reusable type hint experience. Our previous transform now looks like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll: PCollection[int]) -&gt; PCollection[str]:
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>These type hints will actively hook into the internal Beam typing system to
+play a role in pipeline type checking, and runtime type checking.</p>
+<p>So how does this work?</p>
+<h2 id="typed-pcollections">Typed PCollections</h2>
+<p>You guessed it! The PCollection class inherits from <code>typing.Generic</code>, allowing it to be
+parameterized with either zero types (denoted <code>PCollection</code>) or one type (denoted <code>PCollection[T]</code>).</p>
+<ul>
+<li>A PCollection with zero types is implicitly converted to <code>PCollection[Any]</code>.</li>
+<li>A PCollection with one type can have any nested type (e.g. <code>Union[int, str]</code>).</li>
+</ul>
+<p>Internally, Beam&rsquo;s typing system makes these annotations compatible with other
+type hints by removing the outer PCollection container.</p>
+<h2 id="pbegin-pdone-none">PBegin, PDone, None</h2>
+<p>Finally, besides PCollection, a valid annotation on the <code>expand(...)</code> method of a PTransform is
+<code>PBegin</code> or <code>None</code>. These are generally used for PTransforms that begin or end with an I/O operation.</p>
+<p>For instance, when saving data, your transform&rsquo;s output type should be <code>None</code>.</p>
+<pre><code>class SaveResults(beam.PTransform):
+def expand(self, pcoll: PCollection[str]) -&gt; None:
+return pcoll | beam.io.WriteToBigQuery(...)
+</code></pre><h1 id="next-steps">Next Steps</h1>
+<p>What are you waiting for.. start using annotations on your transforms!</p>
+<p>For more background on type hints in Python, see:
+<a href="https://beam.apache.org/documentation/sdks/python-type-safety/">Ensuring Python Type Safety</a>.</p>
+<p>Finally, please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description></item><item><title>Blog: Performance-Driven Runtime Type Checking for the Python SDK</title><link>/blog/python-performance-runtime-type-checking/</link><pubDate>Fri, 21 Aug 2020 00:00:01 -0800</pubDate><guid>/blog/python-performance-runtime-type-checking/</guid><description>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>In this blog post, we&rsquo;re announcing the upcoming release of a new, opt-in
+runtime type checking system for Beam&rsquo;s Python SDK that&rsquo;s optimized for performance
+in both development and production environments.</p>
+<p>But let&rsquo;s take a step back - why do we even care about runtime type checking
+in the first place? Let&rsquo;s look at an example.</p>
+<pre><code>class MultiplyNumberByTwo(beam.DoFn):
+def process(self, element: int):
+return element * 2
+p = Pipeline()
+p | beam.Create(['1', '2'] | beam.ParDo(MultiplyNumberByTwo())
+</code></pre><p>In this code, we passed a list of strings to a DoFn that&rsquo;s clearly intended for use with
+integers. Luckily, this code will throw an error during pipeline construction because
+the inferred output type of <code>beam.Create(['1', '2'])</code> is <code>str</code> which is incompatible with
+the declared input type of <code>MultiplyNumberByTwo.process</code> which is <code>int</code>.</p>
+<p>However, what if we turned pipeline type checking off using the <code>no_pipeline_type_check</code>
+flag? Or more realistically, what if the input PCollection to <code>MultiplyNumberByTwo</code> arrived
+from a database, meaning that the output data type can only be known at runtime?</p>
+<p>In either case, no error would be thrown during pipeline construction.
+And even at runtime, this code works. Each string would be multiplied by 2,
+yielding a result of <code>['11', '22']</code>, but that&rsquo;s certainly not the outcome we want.</p>
+<p>So how do you debug this breed of &ldquo;hidden&rdquo; errors? More broadly speaking, how do you debug
+any typing or serialization error in Beam?</p>
+<p>The answer is to use runtime type checking.</p>
+<h1 id="runtime-type-checking-rtc">Runtime Type Checking (RTC)</h1>
+<p>This feature works by checking that actual input and output values satisfy the declared
+type constraints during pipeline execution. If you ran the code from before with
+<code>runtime_type_check</code> on, you would receive the following error message:</p>
+<pre><code>Type hint violation for 'ParDo(MultiplyByTwo)': requires &lt;class 'int'&gt; but got &lt;class 'str'&gt; for element
+</code></pre><p>This is an actionable error message - it tells you that either your code has a bug
+or that your declared type hints are incorrect. Sounds simple enough, so what&rsquo;s the catch?</p>
+<p><em>It is soooo slowwwwww.</em> See for yourself.</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal Pipeline</th>
+<th>Runtime Type Checking Pipeline</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+</tr>
+</tbody>
+</table>
+<p>In this micro-benchmark, the pipeline with runtime type checking was over 10x slower,
+with the gap only increasing as our input PCollection increased in size.</p>
+<p>So, is there any production-friendly alternative?</p>
+<h1 id="performance-runtime-type-check">Performance Runtime Type Check</h1>
+<p>There is! We developed a new flag called <code>performance_runtime_type_check</code> that
+minimizes its footprint on the pipeline&rsquo;s time complexity using a combination of</p>
+<ul>
+<li>efficient Cython code,</li>
+<li>smart sampling techniques, and</li>
+<li>optimized mega type-hints.</li>
+</ul>
+<p>So what do the new numbers look like?</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal</th>
+<th>RTC</th>
+<th>Performance RTC</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+<td>5.4 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+<td>11.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+<td>25.5 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+<td>39.4 sec</td>
+</tr>
+</tbody>
+</table>
+<p>On average, the new Performance RTC is 4.4% slower than a normal pipeline whereas the old RTC
+is over 900% slower! Additionally, as the size of the input PCollection increases, the fixed cost
+of setting up the Performance RTC system is spread across each element, decreasing the relative
+impact on the overall pipeline. With 18,001 elements, the difference is less than 1 second.</p>
+<h2 id="how-does-it-work">How does it work?</h2>
+<p>There are three key factors responsible for this upgrade in performance.</p>
+<ol>
+<li>
+<p>Instead of type checking all values, we only type check a subset of values, known as
+a sample in statistics. Initially, we sample a substantial number of elements, but as our
+confidence that the element type won&rsquo;t change over time increases, we reduce our
+sampling rate (up to a fixed minimum).</p>
+</li>
+<li>
+<p>Whereas the old RTC system used heavy wrappers to perform the type check, the new RTC system
+moves the type check to a Cython-optimized, non-decorated portion of the codebase. For reference,
+Cython is a programming language that gives C-like performance to Python code.</p>
+</li>
+<li>
+<p>Finally, we use a single mega type hint to type-check only the output values of transforms
+instead of type-checking both the input and output values separately. This mega typehint is composed of
+the original transform&rsquo;s output type constraints along with all consumer transforms&rsquo; input type
+constraints. Using this mega type hint allows us to reduce overhead while simultaneously allowing
+us to throw <em>more actionable errors</em>. For instance, consider the following error (which was
+generated from the old RTC system):</p>
+</li>
+</ol>
+<pre><code>Runtime type violation detected within ParDo(DownstreamDoFn): Type-hint for argument: 'element' violated. Expected an instance of &lt;class ‘str’&gt;, instead found 9, an instance of &lt;class ‘int’&gt;.
+</code></pre><p>This error tells us that the <code>DownstreamDoFn</code> received an <code>int</code> when it was expecting a <code>str</code>, but doesn&rsquo;t tell us
+who created that <code>int</code> in the first place. Who is the offending upstream transform that&rsquo;s responsible for
+this <code>int</code>? Presumably, <em>that</em> transform&rsquo;s output type hints were too expansive (e.g. <code>Any</code>) or otherwise non-existent because
+no error was thrown during the runtime type check of its output.</p>
+<p>The problem here boils down to a lack of context. If we knew who our consumers were when type
+checking our output, we could simultaneously type check our output value against our output type
+constraints and every consumers&rsquo; input type constraints to know whether there is <em>any</em> possibility
+for a mismatch. This is exactly what the mega type hint does, and it allows us to throw errors
+at the point of declaration rather than the point of exception, saving you valuable time
+while providing higher quality error messages.</p>
+<p>So what would the same error look like using Performance RTC? It&rsquo;s the exact same string but with one additional line:</p>
+<pre><code>[while running 'ParDo(UpstreamDoFn)']
+</code></pre><p>And that&rsquo;s much more actionable for an investigation :)</p>
+<h1 id="next-steps">Next Steps</h1>
+<p>Go play with the new <code>performance_runtime_type_check</code> feature!</p>
+<p>It&rsquo;s in an experimental state so please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description></item><item><title>Blog: Apache Beam 2.23.0</title><link>/blog/beam-2.23.0/</link><pubDate>Wed, 29 Jul 2020 00:00:01 -0800</pubDate><guid>/blog/beam-2.23.0/</guid><description>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -1569,7 +1809,7 @@ limitations under the License.
<h3 id="kotlin">Kotlin</h3>
<div class=language-java>
<div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"> <span class="c1">// String templating
-</span><span class="c1"></span> <span class="n">val</span> <span class="n">filename</span> <span class="o">=</span> <span class="s">&#34;$filenamePrefixForWindow(intervalWindow)-$shardNumber-of-$numShards${outputFileHints.suggestedFilenameSuffix)&#34;</span> </code></pre></div>
+</span><span class="c1"></span> <span class="n">val</span> <span class="n">filename</span> <span class="o">=</span> <span class="s">&#34;$filenamePrefixForWindow(intervalWindow)-$shardNumber-of-$numShards${outputFileHints.suggestedFilenameSuffix)&#34;</span></code></pre></div>
</div>
<h3 id="java-1">Java</h3>
<div class=language-java>
@@ -4940,94 +5180,4 @@ limitations under the License.
<p>With this basic change out of the way we just had to make sure that windows were respected for side inputs and that <code>Combine</code> and <code>GroupByKey</code> correctly handled windows. The tricky part there is the handling of merging windows such as session windows. For these we essentially emulate the behavior of a merging <code>WindowFn</code> in our own code.</p>
<p>After we got side inputs working we could enable the aforementioned suite of tests to check how well the runner behaves with respect to the Beam model. As can be expected there were quite some discrepancies but we managed to resolve them all. In the process, we also slimmed down the runner implementation. For example, we removed all custom translations for sources and sinks and are now relying only on Beam code for these, thereby greatly reducing the maintenance overhead.</p>
<h2 id="summary">Summary</h2>
-<p>We reached a major milestone in adding windowing support to the Flink Batch runner, thereby making it compatible with the Beam model. Because of the large suite of tests that can now be executed on the runner we are also confident about the correctness of the implementation and about it staying that way in the future.</p></description></item><item><title>Blog: Where's my PCollection.map()?</title><link>/blog/where-is-my-pcollection-dot-map/</link><pubDate>Fri, 27 May 2016 09:00: [...]
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<p>Have you ever wondered why Beam has PTransforms for everything instead of having methods on PCollection? Take a look at the history that led to this (and other) design decisions.</p>
-<p>Though Beam is relatively new, its design draws heavily on many years of
-experience with real-world pipelines. One of the primary inspirations is
-<a href="https://ai.google/research/pubs/pub35650">FlumeJava</a>, which is Google&rsquo;s
-internal successor to MapReduce first introduced in 2009.</p>
-<p>The original FlumeJava API has methods like <code>count</code> and <code>parallelDo</code> on the PCollections. Though slightly more succinct, this approach has many disadvantages to extensibility. Every new user to FlumeJava wanted to add transforms, and adding them as methods to PCollection simply doesn&rsquo;t scale well. In contrast, a PCollection in Beam has a single <code>apply</code> method which takes any PTransform as an argument.</p>
-<table class="table">
-<tr>
-<th>FlumeJava</th>
-<th>Beam</th>
-</tr>
-<tr>
-<td><pre>
-PCollection&lt;T&gt; input = …
-PCollection&lt;O&gt; output = input.count()
-.parallelDo(...);
-</pre></td>
-<td><pre>
-PCollection&lt;T&gt; input = …
-PCollection&lt;O&gt; output = input.apply(Count.perElement())
-.apply(ParDo.of(...));
-</pre></td>
-</tr>
-</table>
-<p>This is a more scalable approach for several reasons.</p>
-<h2 id="where-to-draw-the-line">Where to draw the line?</h2>
-<p>Adding methods to PCollection forces a line to be drawn between operations that are &ldquo;useful&rdquo; enough to merit this special treatment and those that are not. It is easy to make the case for flat map, group by key, and combine per key. But what about filter? Count? Approximate count? Approximate quantiles? Most frequent? WriteToMyFavoriteSource? Going too far down this path leads to a single enormous class that contains nearly everything one could want to do. (Flum [...]
-<p>Instead in Beam we&rsquo;ve chosen a style that places all transforms&ndash;whether they be primitive operations, composite operations bundled in the SDK, or part of an external library&ndash;on equal footing. This also facilitates alternative implementations (which may even take different options) that are easily interchangeable.</p>
-<table class="table">
-<tr>
-<th>FlumeJava</th>
-<th>Beam</th>
-</tr>
-<tr>
-<td><pre>
-PCollection&lt;O&gt; output =
-ExternalLibrary.doStuff(
-MyLibrary.transform(input, myArgs)
-.parallelDo(...),
-externalLibArgs);
-</pre></td>
-<td><pre>
-PCollection&lt;O&gt; output = input
-.apply(MyLibrary.transform(myArgs))
-.apply(ParDo.of(...))
-.apply(ExternalLibrary.doStuff(externalLibArgs));
-&nbsp;
-</pre></td>
-</tr>
-</table>
-<h2 id="configurability">Configurability</h2>
-<p>It makes for a fluent style to let values (PCollections) be the objects passed around and manipulated (i.e. the handles to the deferred execution graph), but it is the operations themselves that need to be composable, configurable, and extendable. Using PCollection methods for the operations doesn&rsquo;t scale well here, especially in a language without default or keyword arguments. For example, a ParDo operation can have any number of side inputs and side outputs, or a write [...]
-<h2 id="type-safety">Type Safety</h2>
-<p>Many operations can only be applied to collections whose elements are of a specific type. For example, the GroupByKey operation should only be applied to <code>PCollection&lt;KV&lt;K, V&gt;&gt;</code>s. In Java at least, it&rsquo;s not possible to restrict methods based on the element type parameter alone. In FlumeJava, this led us to add a <code>PTable&lt;K, V&gt;</code> subclassing <code>PCollection&lt;KV&lt;K, V&gt;&gt;& [...]
-<p>This is particularly inconvenient for transforms that produce outputs whose element types are the same as (or related to) their input&rsquo;s element types, requiring extra support to generate the right subclasses (e.g. a filter on a PTable should produce another PTable rather than just a raw PCollection of key-value pairs).</p>
-<p>Using PTransforms allows us to sidestep this entire issue. We can place arbitrary constraints on the context in which a transform may be used based on the type of its inputs; for instance GroupByKey is statically typed to only apply to a <code>PCollection&lt;KV&lt;K, V&gt;&gt;</code>. The way this happens is generalizable to arbitrary shapes, without needing to introduce specialized types like PTable.</p>
-<h2 id="reusability-and-structure">Reusability and Structure</h2>
-<p>Though PTransforms are generally constructed at the site at which they&rsquo;re used, by pulling them out as separate objects one is able to store them and pass them around.</p>
-<p>As pipelines grow and evolve, it is useful to structure your pipeline into modular, often reusable components, and PTransforms allow one to do this nicely in a data-processing pipeline. In addition, modular PTransforms also expose the logical structure of your code to the system (e.g. for monitoring). Of the three different representations of the WordCount pipeline below, only the structured view captures the high-level intent of the pipeline. Letting even the simple operations be [...]
-<img class="center-block" src="/images/blog/simple-wordcount-pipeline.png" alt="Three different visualizations of a simple WordCount pipeline" width="500">
-<div class="text-center">
-<i>Three different visualizations of a simple WordCount pipeline which computes the number of occurrences of every word in a set of text files. The flat view gives the full DAG of all operations performed. The execution view groups operations according to how they're executed, e.g. after performing runner-specific optimizations like function composition. The structured view nests operations according to their grouping in PTransforms.</i>
-</div>
-<h2 id="summary">Summary</h2>
-<p>Although it&rsquo;s tempting to add methods to PCollections, such an approach is not scalable, extensible, or sufficiently expressive. Putting a single apply method on PCollection and all the logic into the operation itself lets us have the best of both worlds, and avoids hard cliffs of complexity by having a single consistent style across simple and complex pipelines, and between predefined and user-defined operations.</p></description></item><item><title>Blog: Dynamic work [...]
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<p>This morning, Eugene and Malo from the Google Cloud Dataflow team posted <a href="https://cloud.google.com/blog/big-data/2016/05/no-shard-left-behind-dynamic-work-rebalancing-in-google-cloud-dataflow"><em>No shard left behind: dynamic work rebalancing in Google Cloud Dataflow</em></a>. This article discusses Cloud Dataflow’s solution to the well-known straggler problem.</p>
-<p>In a large batch processing job with many tasks executing in parallel, some of the tasks &ndash; the stragglers &ndash; can take a much longer time to complete than others, perhaps due to imperfect splitting of the work into parallel chunks when issuing the job. Typically, waiting for stragglers means that the overall job completes later than it should, and may also reserve too many machines that may be underutilized at the end. Cloud Dataflow’s dynamic work rebalancing can [...]
-<p>What I’d like to highlight for the Apache Beam (incubating) community is that Cloud Dataflow’s dynamic work rebalancing is implemented using <em>runner-specific</em> control logic on top of Beam’s <em>runner-independent</em> <a href="https://github.com/apache/beam/blob/9fa97fb2491bc784df53fb0f044409dbbc2af3d7/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java"><code>BoundedSource API</code></a>. Specifically, to steal work from a straggle [...]
\ No newline at end of file
+<p>We reached a major milestone in adding windowing support to the Flink Batch runner, thereby making it compatible with the Beam model. Because of the large suite of tests that can now be executed on the runner we are also confident about the correctness of the implementation and about it staying that way in the future.</p></description></item></channel></rss>
\ No newline at end of file
diff --git a/website/generated-content/blog/python-improved-annotations/index.html b/website/generated-content/blog/python-improved-annotations/index.html
new file mode 100644
index 0000000..0e432fd
--- /dev/null
+++ b/website/generated-content/blog/python-improved-annotations/index.html
@@ -0,0 +1,45 @@
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><title>Improved Annotation Support for the Python SDK</title><meta name=description content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Pa [...]
+<span class=sr-only>Toggle navigation</span>
+<span class=icon-bar></span><span class=icon-bar></span><span class=icon-bar></span></button>
+<a href=/ class=navbar-brand><img alt=Brand style=height:25px src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask closed"></div><div id=navbar class="navbar-container closed"><ul class="nav navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a href=/documentation/>Documentation</a></li><li><a href=/documentation/sdks/java/>Languages</a></li><li><a href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a href=/roadmap/>Roadmap</a></li>< [...]
+•
+Saavan Nanavati</p></header><div class=post-content itemprop=articleBody><p>The importance of static type checking in a dynamically
+typed language like Python is not up for debate. Type hints
+allow developers to leverage a strong typing system to:</p><ul><li>write better code,</li><li>self-document ambiguous programming logic, and</li><li>inform intelligent code completion in IDEs like PyCharm.</li></ul><p>This is why we’re excited to announce upcoming improvements to
+the <code>typehints</code> module of Beam’s Python SDK, including support
+for typed PCollections and Python 3 style annotations on PTransforms.</p><h1 id=improved-annotations>Improved Annotations</h1><p>Today, you have the option to declare type hints on PTransforms using either
+class decorators or inline functions.</p><p>For instance, a PTransform with decorated type hints might look like this:</p><pre><code>@beam.typehints.with_input_types(int)
+@beam.typehints.with_output_types(str)
+class IntToStr(beam.PTransform):
+ def expand(self, pcoll):
+ return pcoll | beam.Map(lambda num: str(num))
+
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>Using inline functions instead, the same transform would look like this:</p><pre><code>class IntToStr(beam.PTransform):
+ def expand(self, pcoll):
+ return pcoll | beam.Map(lambda num: str(num))
+
+strings = numbers | beam.ParDo(IntToStr()).with_input_types(int).with_output_types(str)
+</code></pre><p>Both methods have problems. Class decorators are syntax-heavy,
+requiring two additional lines of code, whereas inline functions provide type hints
+that aren’t reusable across other instances of the same transform. Additionally, both
+methods are incompatible with static type checkers like MyPy.</p><p>With Python 3 annotations however, we can subvert these problems to provide a
+clean and reusable type hint experience. Our previous transform now looks like this:</p><pre><code>class IntToStr(beam.PTransform):
+ def expand(self, pcoll: PCollection[int]) -> PCollection[str]:
+ return pcoll | beam.Map(lambda num: str(num))
+
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>These type hints will actively hook into the internal Beam typing system to
+play a role in pipeline type checking, and runtime type checking.</p><p>So how does this work?</p><h2 id=typed-pcollections>Typed PCollections</h2><p>You guessed it! The PCollection class inherits from <code>typing.Generic</code>, allowing it to be
+parameterized with either zero types (denoted <code>PCollection</code>) or one type (denoted <code>PCollection[T]</code>).</p><ul><li>A PCollection with zero types is implicitly converted to <code>PCollection[Any]</code>.</li><li>A PCollection with one type can have any nested type (e.g. <code>Union[int, str]</code>).</li></ul><p>Internally, Beam’s typing system makes these annotations compatible with other
+type hints by removing the outer PCollection container.</p><h2 id=pbegin-pdone-none>PBegin, PDone, None</h2><p>Finally, besides PCollection, a valid annotation on the <code>expand(...)</code> method of a PTransform is
+<code>PBegin</code> or <code>None</code>. These are generally used for PTransforms that begin or end with an I/O operation.</p><p>For instance, when saving data, your transform’s output type should be <code>None</code>.</p><pre><code>class SaveResults(beam.PTransform):
+ def expand(self, pcoll: PCollection[str]) -> None:
+ return pcoll | beam.io.WriteToBigQuery(...)
+</code></pre><h1 id=next-steps>Next Steps</h1><p>What are you waiting for.. start using annotations on your transforms!</p><p>For more background on type hints in Python, see:
+<a href=https://beam.apache.org/documentation/sdks/python-type-safety/>Ensuring Python Type Safety</a>.</p><p>Finally, please
+<a href=https://beam.apache.org/community/contact-us/>let us know</a>
+if you encounter any issues.</p></div></article></div><footer class=footer><div class=footer__contained><div class=footer__cols><div class=footer__cols__col><div class=footer__cols__col__logo><img src=/images/beam_logo_circle.svg class=footer__logo alt="Beam logo"></div><div class=footer__cols__col__logo><img src=/images/apache_logo_circle.svg class=footer__logo alt="Apache logo"></div></div><div class="footer__cols__col footer__cols__col--md"><div class=footer__cols__col__title>Start</d [...]
+<a href=http://www.apache.org>The Apache Software Foundation</a>
+| <a href=/privacy_policy>Privacy Policy</a>
+| <a href=/feed.xml>RSS Feed</a><br><br>Apache Beam, Apache, Beam, the Beam logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation. All other products or name brands are trademarks of their respective holders, including The Apache Software Foundation.</div></footer></body></html>
\ No newline at end of file
diff --git a/website/generated-content/blog/python-performance-runtime-type-checking/index.html b/website/generated-content/blog/python-performance-runtime-type-checking/index.html
new file mode 100644
index 0000000..cbce35a
--- /dev/null
+++ b/website/generated-content/blog/python-performance-runtime-type-checking/index.html
@@ -0,0 +1,57 @@
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><title>Performance-Driven Runtime Type Checking for the Python SDK</title><meta name=description content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise I [...]
+<span class=sr-only>Toggle navigation</span>
+<span class=icon-bar></span><span class=icon-bar></span><span class=icon-bar></span></button>
+<a href=/ class=navbar-brand><img alt=Brand style=height:25px src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask closed"></div><div id=navbar class="navbar-container closed"><ul class="nav navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a href=/documentation/>Documentation</a></li><li><a href=/documentation/sdks/java/>Languages</a></li><li><a href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a href=/roadmap/>Roadmap</a></li>< [...]
+•
+Saavan Nanavati</p></header><div class=post-content itemprop=articleBody><p>In this blog post, we’re announcing the upcoming release of a new, opt-in
+runtime type checking system for Beam’s Python SDK that’s optimized for performance
+in both development and production environments.</p><p>But let’s take a step back - why do we even care about runtime type checking
+in the first place? Let’s look at an example.</p><pre><code>class MultiplyNumberByTwo(beam.DoFn):
+ def process(self, element: int):
+ return element * 2
+
+p = Pipeline()
+p | beam.Create(['1', '2'] | beam.ParDo(MultiplyNumberByTwo())
+</code></pre><p>In this code, we passed a list of strings to a DoFn that’s clearly intended for use with
+integers. Luckily, this code will throw an error during pipeline construction because
+the inferred output type of <code>beam.Create(['1', '2'])</code> is <code>str</code> which is incompatible with
+the declared input type of <code>MultiplyNumberByTwo.process</code> which is <code>int</code>.</p><p>However, what if we turned pipeline type checking off using the <code>no_pipeline_type_check</code>
+flag? Or more realistically, what if the input PCollection to <code>MultiplyNumberByTwo</code> arrived
+from a database, meaning that the output data type can only be known at runtime?</p><p>In either case, no error would be thrown during pipeline construction.
+And even at runtime, this code works. Each string would be multiplied by 2,
+yielding a result of <code>['11', '22']</code>, but that’s certainly not the outcome we want.</p><p>So how do you debug this breed of “hidden” errors? More broadly speaking, how do you debug
+any typing or serialization error in Beam?</p><p>The answer is to use runtime type checking.</p><h1 id=runtime-type-checking-rtc>Runtime Type Checking (RTC)</h1><p>This feature works by checking that actual input and output values satisfy the declared
+type constraints during pipeline execution. If you ran the code from before with
+<code>runtime_type_check</code> on, you would receive the following error message:</p><pre><code>Type hint violation for 'ParDo(MultiplyByTwo)': requires <class 'int'> but got <class 'str'> for element
+</code></pre><p>This is an actionable error message - it tells you that either your code has a bug
+or that your declared type hints are incorrect. Sounds simple enough, so what’s the catch?</p><p><em>It is soooo slowwwwww.</em> See for yourself.</p><table><thead><tr><th>Element Size</th><th>Normal Pipeline</th><th>Runtime Type Checking Pipeline</th></tr></thead><tbody><tr><td>1</td><td>5.3 sec</td><td>5.6 sec</td></tr><tr><td>2,001</td><td>9.4 sec</td><td>57.2 sec</td></tr><tr><td>10,001</td><td>24.5 sec</td><td>259.8 sec</td></tr><tr><td>18,001</td><td>38.7 sec</td><td>450.5 se [...]
+with the gap only increasing as our input PCollection increased in size.</p><p>So, is there any production-friendly alternative?</p><h1 id=performance-runtime-type-check>Performance Runtime Type Check</h1><p>There is! We developed a new flag called <code>performance_runtime_type_check</code> that
+minimizes its footprint on the pipeline’s time complexity using a combination of</p><ul><li>efficient Cython code,</li><li>smart sampling techniques, and</li><li>optimized mega type-hints.</li></ul><p>So what do the new numbers look like?</p><table><thead><tr><th>Element Size</th><th>Normal</th><th>RTC</th><th>Performance RTC</th></tr></thead><tbody><tr><td>1</td><td>5.3 sec</td><td>5.6 sec</td><td>5.4 sec</td></tr><tr><td>2,001</td><td>9.4 sec</td><td>57.2 sec</td><td>11.2 sec</td [...]
+is over 900% slower! Additionally, as the size of the input PCollection increases, the fixed cost
+of setting up the Performance RTC system is spread across each element, decreasing the relative
+impact on the overall pipeline. With 18,001 elements, the difference is less than 1 second.</p><h2 id=how-does-it-work>How does it work?</h2><p>There are three key factors responsible for this upgrade in performance.</p><ol><li><p>Instead of type checking all values, we only type check a subset of values, known as
+a sample in statistics. Initially, we sample a substantial number of elements, but as our
+confidence that the element type won’t change over time increases, we reduce our
+sampling rate (up to a fixed minimum).</p></li><li><p>Whereas the old RTC system used heavy wrappers to perform the type check, the new RTC system
+moves the type check to a Cython-optimized, non-decorated portion of the codebase. For reference,
+Cython is a programming language that gives C-like performance to Python code.</p></li><li><p>Finally, we use a single mega type hint to type-check only the output values of transforms
+instead of type-checking both the input and output values separately. This mega typehint is composed of
+the original transform’s output type constraints along with all consumer transforms’ input type
+constraints. Using this mega type hint allows us to reduce overhead while simultaneously allowing
+us to throw <em>more actionable errors</em>. For instance, consider the following error (which was
+generated from the old RTC system):</p></li></ol><pre><code>Runtime type violation detected within ParDo(DownstreamDoFn): Type-hint for argument: 'element' violated. Expected an instance of <class ‘str’>, instead found 9, an instance of <class ‘int’>.
+</code></pre><p>This error tells us that the <code>DownstreamDoFn</code> received an <code>int</code> when it was expecting a <code>str</code>, but doesn’t tell us
+who created that <code>int</code> in the first place. Who is the offending upstream transform that’s responsible for
+this <code>int</code>? Presumably, <em>that</em> transform’s output type hints were too expansive (e.g. <code>Any</code>) or otherwise non-existent because
+no error was thrown during the runtime type check of its output.</p><p>The problem here boils down to a lack of context. If we knew who our consumers were when type
+checking our output, we could simultaneously type check our output value against our output type
+constraints and every consumers’ input type constraints to know whether there is <em>any</em> possibility
+for a mismatch. This is exactly what the mega type hint does, and it allows us to throw errors
+at the point of declaration rather than the point of exception, saving you valuable time
+while providing higher quality error messages.</p><p>So what would the same error look like using Performance RTC? It’s the exact same string but with one additional line:</p><pre><code>[while running 'ParDo(UpstreamDoFn)']
+</code></pre><p>And that’s much more actionable for an investigation :)</p><h1 id=next-steps>Next Steps</h1><p>Go play with the new <code>performance_runtime_type_check</code> feature!</p><p>It’s in an experimental state so please
+<a href=https://beam.apache.org/community/contact-us/>let us know</a>
+if you encounter any issues.</p></div></article></div><footer class=footer><div class=footer__contained><div class=footer__cols><div class=footer__cols__col><div class=footer__cols__col__logo><img src=/images/beam_logo_circle.svg class=footer__logo alt="Beam logo"></div><div class=footer__cols__col__logo><img src=/images/apache_logo_circle.svg class=footer__logo alt="Apache logo"></div></div><div class="footer__cols__col footer__cols__col--md"><div class=footer__cols__col__title>Start</d [...]
+<a href=http://www.apache.org>The Apache Software Foundation</a>
+| <a href=/privacy_policy>Privacy Policy</a>
+| <a href=/feed.xml>RSS Feed</a><br><br>Apache Beam, Apache, Beam, the Beam logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation. All other products or name brands are trademarks of their respective holders, including The Apache Software Foundation.</div></footer></body></html>
\ No newline at end of file
diff --git a/website/generated-content/blog/splittable-do-fn/index.html b/website/generated-content/blog/splittable-do-fn/index.html
index e22b91d..f6950bc 100644
--- a/website/generated-content/blog/splittable-do-fn/index.html
+++ b/website/generated-content/blog/splittable-do-fn/index.html
@@ -204,7 +204,7 @@ produces pairs <em>(x, 0), (x, 1), …, (x, N-1)</em> as output.</p><div class=l
<span class=k>if</span> <span class=ow>not</span> <span class=n>tracker</span><span class=o>.</span><span class=n>try_claim</span><span class=p>(</span><span class=n>i</span><span class=p>):</span>
<span class=k>return</span>
<span class=k>yield</span> <span class=n>element</span><span class=p>[</span><span class=mi>0</span><span class=p>],</span> <span class=n>i</span>
-
+
<span class=k>def</span> <span class=nf>get_initial_restriction</span><span class=p>(</span><span class=n>element</span><span class=p>):</span>
<span class=k>return</span> <span class=p>(</span><span class=mi>0</span><span class=p>,</span> <span class=n>element</span><span class=p>[</span><span class=mi>1</span><span class=p>])</span></code></pre></div></div><p>This short <code>DoFn</code> subsumes the functionality of
<a href=https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CountingSource.java>CountingSource</a>,
@@ -259,7 +259,7 @@ to illustrate the approach.</p><div class=language-java><div class=highlight><pr
<span class=k>for</span> <span class=n>record</span> <span class=ow>in</span> <span class=n>block</span><span class=o>.</span><span class=n>records</span><span class=p>():</span>
<span class=k>yield</span> <span class=n>record</span>
<span class=n>block</span> <span class=o>=</span> <span class=n>AvroUtils</span><span class=o>.</span><span class=n>get_next_block</span><span class=p>(</span><span class=nb>file</span><span class=p>)</span>
-
+
<span class=k>def</span> <span class=nf>get_initial_restriction</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=n>filename</span><span class=p>):</span>
<span class=k>return</span> <span class=p>(</span><span class=mi>0</span><span class=p>,</span> <span class=n>fileio</span><span class=o>.</span><span class=n>ChannelFactory</span><span class=o>.</span><span class=n>size_in_bytes</span><span class=p>(</span><span class=n>filename</span><span class=p>))</span></code></pre></div></div><p>This hypothetical <code>DoFn</code> reads records from a single Avro file. Notably missing
is the code for expanding a filepattern: it no longer needs to be part of this
diff --git a/website/generated-content/blog/stateful-processing/index.html b/website/generated-content/blog/stateful-processing/index.html
index 17e40b7..67259ad 100644
--- a/website/generated-content/blog/stateful-processing/index.html
+++ b/website/generated-content/blog/stateful-processing/index.html
@@ -28,7 +28,7 @@ for performing an operation in parallel across all elements, and <em><code>Group
for aggregating elements to which you have assigned the same key. In the
picture below (featured in many of our presentations) the color indicates the
key of the element. Thus the <code>GroupByKey</code>/<code>CombinePerKey</code> transform gathers all the
-green squares to produce a single output element.</p><p><img class=center-block src=/images/blog/stateful-processing/pardo-and-gbk.png alt="ParDo and GroupByKey/CombinePerKey:
+green squares to produce a single output element.</p><p><img class=center-block src=/images/blog/stateful-processing/pardo-and-gbk.png alt="ParDo and GroupByKey/CombinePerKey:
Elementwise versus aggregating computations" width=400></p><p>But not all use cases are easily expressed as pipelines of simple <code>ParDo</code>/<code>Map</code> and
<code>GroupByKey</code>/<code>CombinePerKey</code> transforms. The topic of this blog post is a new
extension to the Beam programming model: <strong>per-element operation augmented with
@@ -47,7 +47,7 @@ that it applies to each element. Without stateful augmentations, a <code>DoFn</c
mostly-pure function from inputs to one or more outputs, corresponding to the
Mapper in a MapReduce. With state, a <code>DoFn</code> has the ability to access
persistent mutable state while processing each input element. Consider this
-illustration:</p><p><img class=center-block src=/images/blog/stateful-processing/stateful-dofn.png alt="Stateful DoFn -
+illustration:</p><p><img class=center-block src=/images/blog/stateful-processing/stateful-dofn.png alt="Stateful DoFn -
the runner controls input but the DoFn controls storage and output" width=300></p><p>The first thing to note is that all the data - the little squares, circles, and
triangles - are red. This is to illustrate that stateful processing occurs in
the context of a single key - all of the elements are key-value pairs with the
@@ -231,7 +231,7 @@ against it, outputting the prediction, like so:</p><div class=language-java><div
<span class=n>Model</span> <span class=n>model</span> <span class=o>=</span> <span class=n>ctx</span><span class=o>.</span><span class=na>sideinput</span><span class=o>(</span><span class=n>userModels</span><span class=o>).</span><span class=na>get</span><span class=o>(</span><span class=n>userId</span><span class=o>);</span>
<span class=c1>// Perhaps some logic around when to output a new prediction
-</span><span class=c1></span> <span class=err>…</span> <span class=n>c</span><span class=o>.</span><span class=na>output</span><span class=o>(</span><span class=n>KV</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=n>userId</span><span class=o>,</span> <span class=n>model</span><span class=o>.</span><span class=na>prediction</span><span class=o>(</span><span class=n>event</span><span class=o>)))</span> <span class=err>…</span>
+</span><span class=c1></span> <span class=err>…</span> <span class=n>c</span><span class=o>.</span><span class=na>output</span><span class=o>(</span><span class=n>KV</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=n>userId</span><span class=o>,</span> <span class=n>model</span><span class=o>.</span><span class=na>prediction</span><span class=o>(</span><span class=n>event</span><span class=o>)))</span> <span class=err>…</span>
<span class=o>}</span>
<span class=o>}));</span></code></pre></div></div><div class=language-py><div class=highlight><pre class=chroma><code class=language-py data-lang=py><span class=c1># Events is a collection of (user, event) pairs.</span>
<span class=n>events</span> <span class=o>=</span> <span class=p>(</span><span class=n>p</span> <span class=o>|</span> <span class=n>ReadFromEventSource</span><span class=p>()</span> <span class=o>|</span> <span class=n>beam</span><span class=o>.</span><span class=n>WindowInto</span><span class=p>(</span><span class=o>....</span><span class=p>))</span>
@@ -316,7 +316,7 @@ only features I have already introduced:</p><div class=language-java><div class=
<span class=n>Prediction</span> <span class=n>newPrediction</span> <span class=o>=</span> <span class=n>model</span><span class=o>.</span><span class=na>prediction</span><span class=o>(</span><span class=n>event</span><span class=o>);</span>
<span class=n>model</span><span class=o>.</span><span class=na>add</span><span class=o>(</span><span class=n>event</span><span class=o>);</span>
<span class=n>modelState</span><span class=o>.</span><span class=na>write</span><span class=o>(</span><span class=n>model</span><span class=o>);</span>
- <span class=k>if</span> <span class=o>(</span><span class=n>previousPrediction</span> <span class=o>==</span> <span class=kc>null</span>
+ <span class=k>if</span> <span class=o>(</span><span class=n>previousPrediction</span> <span class=o>==</span> <span class=kc>null</span>
<span class=o>||</span> <span class=n>shouldOutputNewPrediction</span><span class=o>(</span><span class=n>previousPrediction</span><span class=o>,</span> <span class=n>newPrediction</span><span class=o>))</span> <span class=o>{</span>
<span class=n>c</span><span class=o>.</span><span class=na>output</span><span class=o>(</span><span class=n>KV</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=n>userId</span><span class=o>,</span> <span class=n>newPrediction</span><span class=o>));</span>
<span class=n>previousPredictionState</span><span class=o>.</span><span class=na>write</span><span class=o>(</span><span class=n>newPrediction</span><span class=o>);</span>
diff --git a/website/generated-content/blog/timely-processing/index.html b/website/generated-content/blog/timely-processing/index.html
index edca181..c17aa8f 100644
--- a/website/generated-content/blog/timely-processing/index.html
+++ b/website/generated-content/blog/timely-processing/index.html
@@ -88,7 +88,7 @@ elements we have buffered. Here are the state cells in code:</p><div class=langu
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"count"</span><span class=o>)</span>
<span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>ValueState</span><span class=o><</span><span class=n>Integer</span><span class=o>>></span> <span class=n>countState</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>value</span><span class=o>();</span>
- <span class=err>…</span> <span class=n>TBD</span> <span class=err>…</span>
+ <span class=err>…</span> <span class=n>TBD</span> <span class=err>…</span>
<span class=o>}</span></code></pre></div></div><div class=language-py><div class=highlight><pre class=chroma><code class=language-py data-lang=py><span class=k>class</span> <span class=nc>StatefulBufferingFn</span><span class=p>(</span><span class=n>beam</span><span class=o>.</span><span class=n>DoFn</span><span class=p>):</span>
<span class=n>BUFFER_STATE</span> <span class=o>=</span> <span class=n>BagStateSpec</span><span class=p>(</span><span class=s1>'buffer'</span><span class=p>,</span> <span class=n>EventCoder</span><span class=p>())</span>
@@ -128,7 +128,7 @@ events, and output.</p><div class=language-java><div class=highlight><pre class=
<span class=o>}</span>
<span class=o>}</span>
- <span class=err>…</span> <span class=n>TBD</span> <span class=err>…</span>
+ <span class=err>…</span> <span class=n>TBD</span> <span class=err>…</span>
<span class=o>}</span></code></pre></div></div><div class=language-py><div class=highlight><pre class=chroma><code class=language-py data-lang=py><span class=k>class</span> <span class=nc>StatefulBufferingFn</span><span class=p>(</span><span class=n>beam</span><span class=o>.</span><span class=n>DoFn</span><span class=p>):</span>
<span class=n>MAX_BUFFER_SIZE</span> <span class=o>=</span> <span class=mi>500</span><span class=p>;</span>
diff --git a/website/generated-content/categories/blog/index.xml b/website/generated-content/categories/blog/index.xml
index 0a4e6d0..b424dd2 100644
--- a/website/generated-content/categories/blog/index.xml
+++ b/website/generated-content/categories/blog/index.xml
@@ -1,4 +1,244 @@
-<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – blog</title><link>/categories/blog/</link><description>Recent content in blog on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Wed, 29 Jul 2020 00:00:01 -0800</lastBuildDate><atom:link href="/categories/blog/index.xml" rel="self" type="application/rss+xml"/><item><title>Blog: Apache Beam 2.23.0</title><link>/blog/beam-2.23.0/</link><pubDate>Wed, 29 Jul 2020 00:00:01 [...]
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – blog</title><link>/categories/blog/</link><description>Recent content in blog on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Fri, 21 Aug 2020 00:00:01 -0800</lastBuildDate><atom:link href="/categories/blog/index.xml" rel="self" type="application/rss+xml"/><item><title>Blog: Improved Annotation Support for the Python SDK</title><link>/blog/python-improved-annotatio [...]
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>The importance of static type checking in a dynamically
+typed language like Python is not up for debate. Type hints
+allow developers to leverage a strong typing system to:</p>
+<ul>
+<li>write better code,</li>
+<li>self-document ambiguous programming logic, and</li>
+<li>inform intelligent code completion in IDEs like PyCharm.</li>
+</ul>
+<p>This is why we&rsquo;re excited to announce upcoming improvements to
+the <code>typehints</code> module of Beam&rsquo;s Python SDK, including support
+for typed PCollections and Python 3 style annotations on PTransforms.</p>
+<h1 id="improved-annotations">Improved Annotations</h1>
+<p>Today, you have the option to declare type hints on PTransforms using either
+class decorators or inline functions.</p>
+<p>For instance, a PTransform with decorated type hints might look like this:</p>
+<pre><code>@beam.typehints.with_input_types(int)
+@beam.typehints.with_output_types(str)
+class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>Using inline functions instead, the same transform would look like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr()).with_input_types(int).with_output_types(str)
+</code></pre><p>Both methods have problems. Class decorators are syntax-heavy,
+requiring two additional lines of code, whereas inline functions provide type hints
+that aren&rsquo;t reusable across other instances of the same transform. Additionally, both
+methods are incompatible with static type checkers like MyPy.</p>
+<p>With Python 3 annotations however, we can subvert these problems to provide a
+clean and reusable type hint experience. Our previous transform now looks like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll: PCollection[int]) -&gt; PCollection[str]:
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>These type hints will actively hook into the internal Beam typing system to
+play a role in pipeline type checking, and runtime type checking.</p>
+<p>So how does this work?</p>
+<h2 id="typed-pcollections">Typed PCollections</h2>
+<p>You guessed it! The PCollection class inherits from <code>typing.Generic</code>, allowing it to be
+parameterized with either zero types (denoted <code>PCollection</code>) or one type (denoted <code>PCollection[T]</code>).</p>
+<ul>
+<li>A PCollection with zero types is implicitly converted to <code>PCollection[Any]</code>.</li>
+<li>A PCollection with one type can have any nested type (e.g. <code>Union[int, str]</code>).</li>
+</ul>
+<p>Internally, Beam&rsquo;s typing system makes these annotations compatible with other
+type hints by removing the outer PCollection container.</p>
+<h2 id="pbegin-pdone-none">PBegin, PDone, None</h2>
+<p>Finally, besides PCollection, a valid annotation on the <code>expand(...)</code> method of a PTransform is
+<code>PBegin</code> or <code>None</code>. These are generally used for PTransforms that begin or end with an I/O operation.</p>
+<p>For instance, when saving data, your transform&rsquo;s output type should be <code>None</code>.</p>
+<pre><code>class SaveResults(beam.PTransform):
+def expand(self, pcoll: PCollection[str]) -&gt; None:
+return pcoll | beam.io.WriteToBigQuery(...)
+</code></pre><h1 id="next-steps">Next Steps</h1>
+<p>What are you waiting for.. start using annotations on your transforms!</p>
+<p>For more background on type hints in Python, see:
+<a href="https://beam.apache.org/documentation/sdks/python-type-safety/">Ensuring Python Type Safety</a>.</p>
+<p>Finally, please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description></item><item><title>Blog: Performance-Driven Runtime Type Checking for the Python SDK</title><link>/blog/python-performance-runtime-type-checking/</link><pubDate>Fri, 21 Aug 2020 00:00:01 -0800</pubDate><guid>/blog/python-performance-runtime-type-checking/</guid><description>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>In this blog post, we&rsquo;re announcing the upcoming release of a new, opt-in
+runtime type checking system for Beam&rsquo;s Python SDK that&rsquo;s optimized for performance
+in both development and production environments.</p>
+<p>But let&rsquo;s take a step back - why do we even care about runtime type checking
+in the first place? Let&rsquo;s look at an example.</p>
+<pre><code>class MultiplyNumberByTwo(beam.DoFn):
+def process(self, element: int):
+return element * 2
+p = Pipeline()
+p | beam.Create(['1', '2'] | beam.ParDo(MultiplyNumberByTwo())
+</code></pre><p>In this code, we passed a list of strings to a DoFn that&rsquo;s clearly intended for use with
+integers. Luckily, this code will throw an error during pipeline construction because
+the inferred output type of <code>beam.Create(['1', '2'])</code> is <code>str</code> which is incompatible with
+the declared input type of <code>MultiplyNumberByTwo.process</code> which is <code>int</code>.</p>
+<p>However, what if we turned pipeline type checking off using the <code>no_pipeline_type_check</code>
+flag? Or more realistically, what if the input PCollection to <code>MultiplyNumberByTwo</code> arrived
+from a database, meaning that the output data type can only be known at runtime?</p>
+<p>In either case, no error would be thrown during pipeline construction.
+And even at runtime, this code works. Each string would be multiplied by 2,
+yielding a result of <code>['11', '22']</code>, but that&rsquo;s certainly not the outcome we want.</p>
+<p>So how do you debug this breed of &ldquo;hidden&rdquo; errors? More broadly speaking, how do you debug
+any typing or serialization error in Beam?</p>
+<p>The answer is to use runtime type checking.</p>
+<h1 id="runtime-type-checking-rtc">Runtime Type Checking (RTC)</h1>
+<p>This feature works by checking that actual input and output values satisfy the declared
+type constraints during pipeline execution. If you ran the code from before with
+<code>runtime_type_check</code> on, you would receive the following error message:</p>
+<pre><code>Type hint violation for 'ParDo(MultiplyByTwo)': requires &lt;class 'int'&gt; but got &lt;class 'str'&gt; for element
+</code></pre><p>This is an actionable error message - it tells you that either your code has a bug
+or that your declared type hints are incorrect. Sounds simple enough, so what&rsquo;s the catch?</p>
+<p><em>It is soooo slowwwwww.</em> See for yourself.</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal Pipeline</th>
+<th>Runtime Type Checking Pipeline</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+</tr>
+</tbody>
+</table>
+<p>In this micro-benchmark, the pipeline with runtime type checking was over 10x slower,
+with the gap only increasing as our input PCollection increased in size.</p>
+<p>So, is there any production-friendly alternative?</p>
+<h1 id="performance-runtime-type-check">Performance Runtime Type Check</h1>
+<p>There is! We developed a new flag called <code>performance_runtime_type_check</code> that
+minimizes its footprint on the pipeline&rsquo;s time complexity using a combination of</p>
+<ul>
+<li>efficient Cython code,</li>
+<li>smart sampling techniques, and</li>
+<li>optimized mega type-hints.</li>
+</ul>
+<p>So what do the new numbers look like?</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal</th>
+<th>RTC</th>
+<th>Performance RTC</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+<td>5.4 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+<td>11.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+<td>25.5 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+<td>39.4 sec</td>
+</tr>
+</tbody>
+</table>
+<p>On average, the new Performance RTC is 4.4% slower than a normal pipeline whereas the old RTC
+is over 900% slower! Additionally, as the size of the input PCollection increases, the fixed cost
+of setting up the Performance RTC system is spread across each element, decreasing the relative
+impact on the overall pipeline. With 18,001 elements, the difference is less than 1 second.</p>
+<h2 id="how-does-it-work">How does it work?</h2>
+<p>There are three key factors responsible for this upgrade in performance.</p>
+<ol>
+<li>
+<p>Instead of type checking all values, we only type check a subset of values, known as
+a sample in statistics. Initially, we sample a substantial number of elements, but as our
+confidence that the element type won&rsquo;t change over time increases, we reduce our
+sampling rate (up to a fixed minimum).</p>
+</li>
+<li>
+<p>Whereas the old RTC system used heavy wrappers to perform the type check, the new RTC system
+moves the type check to a Cython-optimized, non-decorated portion of the codebase. For reference,
+Cython is a programming language that gives C-like performance to Python code.</p>
+</li>
+<li>
+<p>Finally, we use a single mega type hint to type-check only the output values of transforms
+instead of type-checking both the input and output values separately. This mega typehint is composed of
+the original transform&rsquo;s output type constraints along with all consumer transforms&rsquo; input type
+constraints. Using this mega type hint allows us to reduce overhead while simultaneously allowing
+us to throw <em>more actionable errors</em>. For instance, consider the following error (which was
+generated from the old RTC system):</p>
+</li>
+</ol>
+<pre><code>Runtime type violation detected within ParDo(DownstreamDoFn): Type-hint for argument: 'element' violated. Expected an instance of &lt;class ‘str’&gt;, instead found 9, an instance of &lt;class ‘int’&gt;.
+</code></pre><p>This error tells us that the <code>DownstreamDoFn</code> received an <code>int</code> when it was expecting a <code>str</code>, but doesn&rsquo;t tell us
+who created that <code>int</code> in the first place. Who is the offending upstream transform that&rsquo;s responsible for
+this <code>int</code>? Presumably, <em>that</em> transform&rsquo;s output type hints were too expansive (e.g. <code>Any</code>) or otherwise non-existent because
+no error was thrown during the runtime type check of its output.</p>
+<p>The problem here boils down to a lack of context. If we knew who our consumers were when type
+checking our output, we could simultaneously type check our output value against our output type
+constraints and every consumers&rsquo; input type constraints to know whether there is <em>any</em> possibility
+for a mismatch. This is exactly what the mega type hint does, and it allows us to throw errors
+at the point of declaration rather than the point of exception, saving you valuable time
+while providing higher quality error messages.</p>
+<p>So what would the same error look like using Performance RTC? It&rsquo;s the exact same string but with one additional line:</p>
+<pre><code>[while running 'ParDo(UpstreamDoFn)']
+</code></pre><p>And that&rsquo;s much more actionable for an investigation :)</p>
+<h1 id="next-steps">Next Steps</h1>
+<p>Go play with the new <code>performance_runtime_type_check</code> feature!</p>
+<p>It&rsquo;s in an experimental state so please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description></item><item><title>Blog: Apache Beam 2.23.0</title><link>/blog/beam-2.23.0/</link><pubDate>Wed, 29 Jul 2020 00:00:01 -0800</pubDate><guid>/blog/beam-2.23.0/</guid><description>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -1569,7 +1809,7 @@ limitations under the License.
<h3 id="kotlin">Kotlin</h3>
<div class=language-java>
<div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"> <span class="c1">// String templating
-</span><span class="c1"></span> <span class="n">val</span> <span class="n">filename</span> <span class="o">=</span> <span class="s">&#34;$filenamePrefixForWindow(intervalWindow)-$shardNumber-of-$numShards${outputFileHints.suggestedFilenameSuffix)&#34;</span> </code></pre></div>
+</span><span class="c1"></span> <span class="n">val</span> <span class="n">filename</span> <span class="o">=</span> <span class="s">&#34;$filenamePrefixForWindow(intervalWindow)-$shardNumber-of-$numShards${outputFileHints.suggestedFilenameSuffix)&#34;</span></code></pre></div>
</div>
<h3 id="java-1">Java</h3>
<div class=language-java>
diff --git a/website/generated-content/categories/index.xml b/website/generated-content/categories/index.xml
index bf900b4..b9555ba 100644
--- a/website/generated-content/categories/index.xml
+++ b/website/generated-content/categories/index.xml
@@ -1 +1 @@
-<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – Categories</title><link>/categories/</link><description>Recent content in Categories on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Wed, 29 Jul 2020 00:00:01 -0800</lastBuildDate><atom:link href="/categories/index.xml" rel="self" type="application/rss+xml"/></channel></rss>
\ No newline at end of file
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – Categories</title><link>/categories/</link><description>Recent content in Categories on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Fri, 21 Aug 2020 00:00:01 -0800</lastBuildDate><atom:link href="/categories/index.xml" rel="self" type="application/rss+xml"/></channel></rss>
\ No newline at end of file
diff --git a/website/generated-content/categories/python/index.xml b/website/generated-content/categories/python/index.xml
index 30c645b..cfc2130 100644
--- a/website/generated-content/categories/python/index.xml
+++ b/website/generated-content/categories/python/index.xml
@@ -1,4 +1,244 @@
-<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – python</title><link>/categories/python/</link><description>Recent content in python on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Thu, 28 May 2020 00:00:01 -0800</lastBuildDate><atom:link href="/categories/python/index.xml" rel="self" type="application/rss+xml"/><item><title>Blog: Python SDK Typing Changes</title><link>/blog/python-typing/</link><pubDate>Thu, 28 [...]
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – python</title><link>/categories/python/</link><description>Recent content in python on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Fri, 21 Aug 2020 00:00:01 -0800</lastBuildDate><atom:link href="/categories/python/index.xml" rel="self" type="application/rss+xml"/><item><title>Blog: Improved Annotation Support for the Python SDK</title><link>/blog/python-improved-a [...]
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>The importance of static type checking in a dynamically
+typed language like Python is not up for debate. Type hints
+allow developers to leverage a strong typing system to:</p>
+<ul>
+<li>write better code,</li>
+<li>self-document ambiguous programming logic, and</li>
+<li>inform intelligent code completion in IDEs like PyCharm.</li>
+</ul>
+<p>This is why we&rsquo;re excited to announce upcoming improvements to
+the <code>typehints</code> module of Beam&rsquo;s Python SDK, including support
+for typed PCollections and Python 3 style annotations on PTransforms.</p>
+<h1 id="improved-annotations">Improved Annotations</h1>
+<p>Today, you have the option to declare type hints on PTransforms using either
+class decorators or inline functions.</p>
+<p>For instance, a PTransform with decorated type hints might look like this:</p>
+<pre><code>@beam.typehints.with_input_types(int)
+@beam.typehints.with_output_types(str)
+class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>Using inline functions instead, the same transform would look like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr()).with_input_types(int).with_output_types(str)
+</code></pre><p>Both methods have problems. Class decorators are syntax-heavy,
+requiring two additional lines of code, whereas inline functions provide type hints
+that aren&rsquo;t reusable across other instances of the same transform. Additionally, both
+methods are incompatible with static type checkers like MyPy.</p>
+<p>With Python 3 annotations however, we can subvert these problems to provide a
+clean and reusable type hint experience. Our previous transform now looks like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll: PCollection[int]) -&gt; PCollection[str]:
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>These type hints will actively hook into the internal Beam typing system to
+play a role in pipeline type checking, and runtime type checking.</p>
+<p>So how does this work?</p>
+<h2 id="typed-pcollections">Typed PCollections</h2>
+<p>You guessed it! The PCollection class inherits from <code>typing.Generic</code>, allowing it to be
+parameterized with either zero types (denoted <code>PCollection</code>) or one type (denoted <code>PCollection[T]</code>).</p>
+<ul>
+<li>A PCollection with zero types is implicitly converted to <code>PCollection[Any]</code>.</li>
+<li>A PCollection with one type can have any nested type (e.g. <code>Union[int, str]</code>).</li>
+</ul>
+<p>Internally, Beam&rsquo;s typing system makes these annotations compatible with other
+type hints by removing the outer PCollection container.</p>
+<h2 id="pbegin-pdone-none">PBegin, PDone, None</h2>
+<p>Finally, besides PCollection, a valid annotation on the <code>expand(...)</code> method of a PTransform is
+<code>PBegin</code> or <code>None</code>. These are generally used for PTransforms that begin or end with an I/O operation.</p>
+<p>For instance, when saving data, your transform&rsquo;s output type should be <code>None</code>.</p>
+<pre><code>class SaveResults(beam.PTransform):
+def expand(self, pcoll: PCollection[str]) -&gt; None:
+return pcoll | beam.io.WriteToBigQuery(...)
+</code></pre><h1 id="next-steps">Next Steps</h1>
+<p>What are you waiting for.. start using annotations on your transforms!</p>
+<p>For more background on type hints in Python, see:
+<a href="https://beam.apache.org/documentation/sdks/python-type-safety/">Ensuring Python Type Safety</a>.</p>
+<p>Finally, please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description></item><item><title>Blog: Performance-Driven Runtime Type Checking for the Python SDK</title><link>/blog/python-performance-runtime-type-checking/</link><pubDate>Fri, 21 Aug 2020 00:00:01 -0800</pubDate><guid>/blog/python-performance-runtime-type-checking/</guid><description>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>In this blog post, we&rsquo;re announcing the upcoming release of a new, opt-in
+runtime type checking system for Beam&rsquo;s Python SDK that&rsquo;s optimized for performance
+in both development and production environments.</p>
+<p>But let&rsquo;s take a step back - why do we even care about runtime type checking
+in the first place? Let&rsquo;s look at an example.</p>
+<pre><code>class MultiplyNumberByTwo(beam.DoFn):
+def process(self, element: int):
+return element * 2
+p = Pipeline()
+p | beam.Create(['1', '2'] | beam.ParDo(MultiplyNumberByTwo())
+</code></pre><p>In this code, we passed a list of strings to a DoFn that&rsquo;s clearly intended for use with
+integers. Luckily, this code will throw an error during pipeline construction because
+the inferred output type of <code>beam.Create(['1', '2'])</code> is <code>str</code> which is incompatible with
+the declared input type of <code>MultiplyNumberByTwo.process</code> which is <code>int</code>.</p>
+<p>However, what if we turned pipeline type checking off using the <code>no_pipeline_type_check</code>
+flag? Or more realistically, what if the input PCollection to <code>MultiplyNumberByTwo</code> arrived
+from a database, meaning that the output data type can only be known at runtime?</p>
+<p>In either case, no error would be thrown during pipeline construction.
+And even at runtime, this code works. Each string would be multiplied by 2,
+yielding a result of <code>['11', '22']</code>, but that&rsquo;s certainly not the outcome we want.</p>
+<p>So how do you debug this breed of &ldquo;hidden&rdquo; errors? More broadly speaking, how do you debug
+any typing or serialization error in Beam?</p>
+<p>The answer is to use runtime type checking.</p>
+<h1 id="runtime-type-checking-rtc">Runtime Type Checking (RTC)</h1>
+<p>This feature works by checking that actual input and output values satisfy the declared
+type constraints during pipeline execution. If you ran the code from before with
+<code>runtime_type_check</code> on, you would receive the following error message:</p>
+<pre><code>Type hint violation for 'ParDo(MultiplyByTwo)': requires &lt;class 'int'&gt; but got &lt;class 'str'&gt; for element
+</code></pre><p>This is an actionable error message - it tells you that either your code has a bug
+or that your declared type hints are incorrect. Sounds simple enough, so what&rsquo;s the catch?</p>
+<p><em>It is soooo slowwwwww.</em> See for yourself.</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal Pipeline</th>
+<th>Runtime Type Checking Pipeline</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+</tr>
+</tbody>
+</table>
+<p>In this micro-benchmark, the pipeline with runtime type checking was over 10x slower,
+with the gap only increasing as our input PCollection increased in size.</p>
+<p>So, is there any production-friendly alternative?</p>
+<h1 id="performance-runtime-type-check">Performance Runtime Type Check</h1>
+<p>There is! We developed a new flag called <code>performance_runtime_type_check</code> that
+minimizes its footprint on the pipeline&rsquo;s time complexity using a combination of</p>
+<ul>
+<li>efficient Cython code,</li>
+<li>smart sampling techniques, and</li>
+<li>optimized mega type-hints.</li>
+</ul>
+<p>So what do the new numbers look like?</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal</th>
+<th>RTC</th>
+<th>Performance RTC</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+<td>5.4 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+<td>11.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+<td>25.5 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+<td>39.4 sec</td>
+</tr>
+</tbody>
+</table>
+<p>On average, the new Performance RTC is 4.4% slower than a normal pipeline whereas the old RTC
+is over 900% slower! Additionally, as the size of the input PCollection increases, the fixed cost
+of setting up the Performance RTC system is spread across each element, decreasing the relative
+impact on the overall pipeline. With 18,001 elements, the difference is less than 1 second.</p>
+<h2 id="how-does-it-work">How does it work?</h2>
+<p>There are three key factors responsible for this upgrade in performance.</p>
+<ol>
+<li>
+<p>Instead of type checking all values, we only type check a subset of values, known as
+a sample in statistics. Initially, we sample a substantial number of elements, but as our
+confidence that the element type won&rsquo;t change over time increases, we reduce our
+sampling rate (up to a fixed minimum).</p>
+</li>
+<li>
+<p>Whereas the old RTC system used heavy wrappers to perform the type check, the new RTC system
+moves the type check to a Cython-optimized, non-decorated portion of the codebase. For reference,
+Cython is a programming language that gives C-like performance to Python code.</p>
+</li>
+<li>
+<p>Finally, we use a single mega type hint to type-check only the output values of transforms
+instead of type-checking both the input and output values separately. This mega typehint is composed of
+the original transform&rsquo;s output type constraints along with all consumer transforms&rsquo; input type
+constraints. Using this mega type hint allows us to reduce overhead while simultaneously allowing
+us to throw <em>more actionable errors</em>. For instance, consider the following error (which was
+generated from the old RTC system):</p>
+</li>
+</ol>
+<pre><code>Runtime type violation detected within ParDo(DownstreamDoFn): Type-hint for argument: 'element' violated. Expected an instance of &lt;class ‘str’&gt;, instead found 9, an instance of &lt;class ‘int’&gt;.
+</code></pre><p>This error tells us that the <code>DownstreamDoFn</code> received an <code>int</code> when it was expecting a <code>str</code>, but doesn&rsquo;t tell us
+who created that <code>int</code> in the first place. Who is the offending upstream transform that&rsquo;s responsible for
+this <code>int</code>? Presumably, <em>that</em> transform&rsquo;s output type hints were too expansive (e.g. <code>Any</code>) or otherwise non-existent because
+no error was thrown during the runtime type check of its output.</p>
+<p>The problem here boils down to a lack of context. If we knew who our consumers were when type
+checking our output, we could simultaneously type check our output value against our output type
+constraints and every consumers&rsquo; input type constraints to know whether there is <em>any</em> possibility
+for a mismatch. This is exactly what the mega type hint does, and it allows us to throw errors
+at the point of declaration rather than the point of exception, saving you valuable time
+while providing higher quality error messages.</p>
+<p>So what would the same error look like using Performance RTC? It&rsquo;s the exact same string but with one additional line:</p>
+<pre><code>[while running 'ParDo(UpstreamDoFn)']
+</code></pre><p>And that&rsquo;s much more actionable for an investigation :)</p>
+<h1 id="next-steps">Next Steps</h1>
+<p>Go play with the new <code>performance_runtime_type_check</code> feature!</p>
+<p>It&rsquo;s in an experimental state so please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description></item><item><title>Blog: Python SDK Typing Changes</title><link>/blog/python-typing/</link><pubDate>Thu, 28 May 2020 00:00:01 -0800</pubDate><guid>/blog/python-typing/</guid><description>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
diff --git a/website/generated-content/categories/typing/index.xml b/website/generated-content/categories/typing/index.xml
index 4e58f4e..00766ef 100644
--- a/website/generated-content/categories/typing/index.xml
+++ b/website/generated-content/categories/typing/index.xml
@@ -1,4 +1,244 @@
-<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – typing</title><link>/categories/typing/</link><description>Recent content in typing on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Thu, 28 May 2020 00:00:01 -0800</lastBuildDate><atom:link href="/categories/typing/index.xml" rel="self" type="application/rss+xml"/><item><title>Blog: Python SDK Typing Changes</title><link>/blog/python-typing/</link><pubDate>Thu, 28 [...]
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Apache Beam – typing</title><link>/categories/typing/</link><description>Recent content in typing on Apache Beam</description><generator>Hugo -- gohugo.io</generator><lastBuildDate>Fri, 21 Aug 2020 00:00:01 -0800</lastBuildDate><atom:link href="/categories/typing/index.xml" rel="self" type="application/rss+xml"/><item><title>Blog: Improved Annotation Support for the Python SDK</title><link>/blog/python-improved-a [...]
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>The importance of static type checking in a dynamically
+typed language like Python is not up for debate. Type hints
+allow developers to leverage a strong typing system to:</p>
+<ul>
+<li>write better code,</li>
+<li>self-document ambiguous programming logic, and</li>
+<li>inform intelligent code completion in IDEs like PyCharm.</li>
+</ul>
+<p>This is why we&rsquo;re excited to announce upcoming improvements to
+the <code>typehints</code> module of Beam&rsquo;s Python SDK, including support
+for typed PCollections and Python 3 style annotations on PTransforms.</p>
+<h1 id="improved-annotations">Improved Annotations</h1>
+<p>Today, you have the option to declare type hints on PTransforms using either
+class decorators or inline functions.</p>
+<p>For instance, a PTransform with decorated type hints might look like this:</p>
+<pre><code>@beam.typehints.with_input_types(int)
+@beam.typehints.with_output_types(str)
+class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>Using inline functions instead, the same transform would look like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr()).with_input_types(int).with_output_types(str)
+</code></pre><p>Both methods have problems. Class decorators are syntax-heavy,
+requiring two additional lines of code, whereas inline functions provide type hints
+that aren&rsquo;t reusable across other instances of the same transform. Additionally, both
+methods are incompatible with static type checkers like MyPy.</p>
+<p>With Python 3 annotations however, we can subvert these problems to provide a
+clean and reusable type hint experience. Our previous transform now looks like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll: PCollection[int]) -&gt; PCollection[str]:
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>These type hints will actively hook into the internal Beam typing system to
+play a role in pipeline type checking, and runtime type checking.</p>
+<p>So how does this work?</p>
+<h2 id="typed-pcollections">Typed PCollections</h2>
+<p>You guessed it! The PCollection class inherits from <code>typing.Generic</code>, allowing it to be
+parameterized with either zero types (denoted <code>PCollection</code>) or one type (denoted <code>PCollection[T]</code>).</p>
+<ul>
+<li>A PCollection with zero types is implicitly converted to <code>PCollection[Any]</code>.</li>
+<li>A PCollection with one type can have any nested type (e.g. <code>Union[int, str]</code>).</li>
+</ul>
+<p>Internally, Beam&rsquo;s typing system makes these annotations compatible with other
+type hints by removing the outer PCollection container.</p>
+<h2 id="pbegin-pdone-none">PBegin, PDone, None</h2>
+<p>Finally, besides PCollection, a valid annotation on the <code>expand(...)</code> method of a PTransform is
+<code>PBegin</code> or <code>None</code>. These are generally used for PTransforms that begin or end with an I/O operation.</p>
+<p>For instance, when saving data, your transform&rsquo;s output type should be <code>None</code>.</p>
+<pre><code>class SaveResults(beam.PTransform):
+def expand(self, pcoll: PCollection[str]) -&gt; None:
+return pcoll | beam.io.WriteToBigQuery(...)
+</code></pre><h1 id="next-steps">Next Steps</h1>
+<p>What are you waiting for.. start using annotations on your transforms!</p>
+<p>For more background on type hints in Python, see:
+<a href="https://beam.apache.org/documentation/sdks/python-type-safety/">Ensuring Python Type Safety</a>.</p>
+<p>Finally, please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description></item><item><title>Blog: Performance-Driven Runtime Type Checking for the Python SDK</title><link>/blog/python-performance-runtime-type-checking/</link><pubDate>Fri, 21 Aug 2020 00:00:01 -0800</pubDate><guid>/blog/python-performance-runtime-type-checking/</guid><description>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>In this blog post, we&rsquo;re announcing the upcoming release of a new, opt-in
+runtime type checking system for Beam&rsquo;s Python SDK that&rsquo;s optimized for performance
+in both development and production environments.</p>
+<p>But let&rsquo;s take a step back - why do we even care about runtime type checking
+in the first place? Let&rsquo;s look at an example.</p>
+<pre><code>class MultiplyNumberByTwo(beam.DoFn):
+def process(self, element: int):
+return element * 2
+p = Pipeline()
+p | beam.Create(['1', '2'] | beam.ParDo(MultiplyNumberByTwo())
+</code></pre><p>In this code, we passed a list of strings to a DoFn that&rsquo;s clearly intended for use with
+integers. Luckily, this code will throw an error during pipeline construction because
+the inferred output type of <code>beam.Create(['1', '2'])</code> is <code>str</code> which is incompatible with
+the declared input type of <code>MultiplyNumberByTwo.process</code> which is <code>int</code>.</p>
+<p>However, what if we turned pipeline type checking off using the <code>no_pipeline_type_check</code>
+flag? Or more realistically, what if the input PCollection to <code>MultiplyNumberByTwo</code> arrived
+from a database, meaning that the output data type can only be known at runtime?</p>
+<p>In either case, no error would be thrown during pipeline construction.
+And even at runtime, this code works. Each string would be multiplied by 2,
+yielding a result of <code>['11', '22']</code>, but that&rsquo;s certainly not the outcome we want.</p>
+<p>So how do you debug this breed of &ldquo;hidden&rdquo; errors? More broadly speaking, how do you debug
+any typing or serialization error in Beam?</p>
+<p>The answer is to use runtime type checking.</p>
+<h1 id="runtime-type-checking-rtc">Runtime Type Checking (RTC)</h1>
+<p>This feature works by checking that actual input and output values satisfy the declared
+type constraints during pipeline execution. If you ran the code from before with
+<code>runtime_type_check</code> on, you would receive the following error message:</p>
+<pre><code>Type hint violation for 'ParDo(MultiplyByTwo)': requires &lt;class 'int'&gt; but got &lt;class 'str'&gt; for element
+</code></pre><p>This is an actionable error message - it tells you that either your code has a bug
+or that your declared type hints are incorrect. Sounds simple enough, so what&rsquo;s the catch?</p>
+<p><em>It is soooo slowwwwww.</em> See for yourself.</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal Pipeline</th>
+<th>Runtime Type Checking Pipeline</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+</tr>
+</tbody>
+</table>
+<p>In this micro-benchmark, the pipeline with runtime type checking was over 10x slower,
+with the gap only increasing as our input PCollection increased in size.</p>
+<p>So, is there any production-friendly alternative?</p>
+<h1 id="performance-runtime-type-check">Performance Runtime Type Check</h1>
+<p>There is! We developed a new flag called <code>performance_runtime_type_check</code> that
+minimizes its footprint on the pipeline&rsquo;s time complexity using a combination of</p>
+<ul>
+<li>efficient Cython code,</li>
+<li>smart sampling techniques, and</li>
+<li>optimized mega type-hints.</li>
+</ul>
+<p>So what do the new numbers look like?</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal</th>
+<th>RTC</th>
+<th>Performance RTC</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+<td>5.4 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+<td>11.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+<td>25.5 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+<td>39.4 sec</td>
+</tr>
+</tbody>
+</table>
+<p>On average, the new Performance RTC is 4.4% slower than a normal pipeline whereas the old RTC
+is over 900% slower! Additionally, as the size of the input PCollection increases, the fixed cost
+of setting up the Performance RTC system is spread across each element, decreasing the relative
+impact on the overall pipeline. With 18,001 elements, the difference is less than 1 second.</p>
+<h2 id="how-does-it-work">How does it work?</h2>
+<p>There are three key factors responsible for this upgrade in performance.</p>
+<ol>
+<li>
+<p>Instead of type checking all values, we only type check a subset of values, known as
+a sample in statistics. Initially, we sample a substantial number of elements, but as our
+confidence that the element type won&rsquo;t change over time increases, we reduce our
+sampling rate (up to a fixed minimum).</p>
+</li>
+<li>
+<p>Whereas the old RTC system used heavy wrappers to perform the type check, the new RTC system
+moves the type check to a Cython-optimized, non-decorated portion of the codebase. For reference,
+Cython is a programming language that gives C-like performance to Python code.</p>
+</li>
+<li>
+<p>Finally, we use a single mega type hint to type-check only the output values of transforms
+instead of type-checking both the input and output values separately. This mega typehint is composed of
+the original transform&rsquo;s output type constraints along with all consumer transforms&rsquo; input type
+constraints. Using this mega type hint allows us to reduce overhead while simultaneously allowing
+us to throw <em>more actionable errors</em>. For instance, consider the following error (which was
+generated from the old RTC system):</p>
+</li>
+</ol>
+<pre><code>Runtime type violation detected within ParDo(DownstreamDoFn): Type-hint for argument: 'element' violated. Expected an instance of &lt;class ‘str’&gt;, instead found 9, an instance of &lt;class ‘int’&gt;.
+</code></pre><p>This error tells us that the <code>DownstreamDoFn</code> received an <code>int</code> when it was expecting a <code>str</code>, but doesn&rsquo;t tell us
+who created that <code>int</code> in the first place. Who is the offending upstream transform that&rsquo;s responsible for
+this <code>int</code>? Presumably, <em>that</em> transform&rsquo;s output type hints were too expansive (e.g. <code>Any</code>) or otherwise non-existent because
+no error was thrown during the runtime type check of its output.</p>
+<p>The problem here boils down to a lack of context. If we knew who our consumers were when type
+checking our output, we could simultaneously type check our output value against our output type
+constraints and every consumers&rsquo; input type constraints to know whether there is <em>any</em> possibility
+for a mismatch. This is exactly what the mega type hint does, and it allows us to throw errors
+at the point of declaration rather than the point of exception, saving you valuable time
+while providing higher quality error messages.</p>
+<p>So what would the same error look like using Performance RTC? It&rsquo;s the exact same string but with one additional line:</p>
+<pre><code>[while running 'ParDo(UpstreamDoFn)']
+</code></pre><p>And that&rsquo;s much more actionable for an investigation :)</p>
+<h1 id="next-steps">Next Steps</h1>
+<p>Go play with the new <code>performance_runtime_type_check</code> feature!</p>
+<p>It&rsquo;s in an experimental state so please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description></item><item><title>Blog: Python SDK Typing Changes</title><link>/blog/python-typing/</link><pubDate>Thu, 28 May 2020 00:00:01 -0800</pubDate><guid>/blog/python-typing/</guid><description>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
diff --git a/website/generated-content/contribute/release-guide/index.html b/website/generated-content/contribute/release-guide/index.html
index 002ffb4..e60148a 100644
--- a/website/generated-content/contribute/release-guide/index.html
+++ b/website/generated-content/contribute/release-guide/index.html
@@ -67,7 +67,7 @@ oversees these just like any other JIRA issue marked with the ‘Fix Version
./beam/release/src/main/scripts/cut_release_branch.sh \
--release=${RELEASE_VERSION} \
--next_release=${NEXT_VERSION}
-
+
# Show help page
./beam/release/src/main/scripts/cut_release_branch.sh -h
</code></pre></li><li><p>The script will:</p><ol><li><p>Create release-${RELEASE_VERSION} branch locally.</p></li><li><p>Change and commit dev versoin number in master branch:</p><p><a href=https://github.com/apache/beam/blob/e8abafe360e126818fe80ae0f6075e71f0fc227d/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy#L209>BeamModulePlugin.groovy</a>,
@@ -119,7 +119,7 @@ sudo apt-get install python3-dev
sudo apt-get install python3.5-dev
sudo apt-get install python3.6-dev
sudo apt-get install python3.7-dev
-</code></pre></li></ol></li><li><p>Run gradle release build</p><ol><li><p>Clean current workspace</p><pre><code>git clean -fdx
+</code></pre></li></ol></li><li><p>Run gradle release build</p><ol><li><p>Clean current workspace</p><pre><code>git clean -fdx
./gradlew clean
</code></pre></li><li><p>Unlock the secret key</p><pre><code>gpg --output ~/doc.sig --sign ~/.bashrc
</code></pre></li><li><p>Run build command</p><pre><code>./gradlew build -PisRelease --no-parallel --scan --stacktrace --continue
@@ -268,11 +268,11 @@ Thanks everyone!
-PgcpProject=${YOUR_GCP_PROJECT} \
-PgcsBucket=${YOUR_GCP_BUCKET}
</code></pre></li><li><p>Java Mobile Game(UserScore, HourlyTeamScore, Leaderboard)</p><p>Pre-request</p><ul><li><p>Create your own BigQuery dataset</p><pre><code>bq mk --project_id=${YOUR_GCP_PROJECT} ${YOUR_DATASET}
-</code></pre></li><li><p>Create yout PubSub topic</p><pre><code>gcloud alpha pubsub topics create --project=${YOUR_GCP_PROJECT} ${YOUR_PROJECT_PUBSUB_TOPIC}
+</code></pre></li><li><p>Create yout PubSub topic</p><pre><code>gcloud alpha pubsub topics create --project=${YOUR_GCP_PROJECT} ${YOUR_PROJECT_PUBSUB_TOPIC}
</code></pre></li><li><p>Setup your service account</p><p>Goto IAM console in your project to create a service account as <code>project owner</code></p><p>Run</p><pre><code>gcloud iam service-accounts keys create ${YOUR_KEY_JSON} --iam-account ${YOUR_SERVICE_ACCOUNT_NAME}@${YOUR_PROJECT_NAME}
-export GOOGLE_APPLICATION_CREDENTIALS=${PATH_TO_YOUR_KEY_JSON}
+export GOOGLE_APPLICATION_CREDENTIALS=${PATH_TO_YOUR_KEY_JSON}
</code></pre></li></ul><p>Run</p><pre><code>./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow \
- -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \
+ -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \
-Pver=${RELEASE_VERSION} \
-PgcpProject=${YOUR_GCP_PROJECT} \
-PgcsBucket=${YOUR_GCP_BUCKET} \
@@ -321,33 +321,33 @@ gcloud alpha pubsub topics create --project=${YOUR_PROJECT} ${YOUR_PUBSUB_TOPIC}
-Dpackage=org.apache.beam.examples \
-DinteractiveMode=false
-DarchetypeCatalog=internal
-
+
mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \
-Dexec.args="${YOUR_PROJECT} ${YOUR_PUBSUB_TOPIC} none"
</code></pre></li></ul></li><li><p>Run Leaderboard with Direct Runner</p><pre><code>python -m apache_beam.examples.complete.game.leader_board \
--project=${YOUR_PROJECT} \
--topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \
--dataset ${USER}_test
-</code></pre><p>Inspect results:</p><ul><li>Check whether there is any error messages in console.</li><li>Goto your BigQuery console and check whether your ${USER}_test has leader_board_users and leader_board_teams table.</li><li>bq head -n 10 ${USER}_test.leader_board_users</li><li>bq head -n 10 ${USER}_test.leader_board_teams</li></ul></li><li><p>Run Leaderboard with Dataflow Runner</p><pre><code>python -m apache_beam.examples.complete.game.leader_board \
---project=${YOUR_PROJECT} \
+</code></pre><p>Inspect results:</p><ul><li>Check whether there is any error messages in console.</li><li>Goto your BigQuery console and check whether your ${USER}_test has leader_board_users and leader_board_teams table.</li><li>bq head -n 10 ${USER}_test.leader_board_users</li><li>bq head -n 10 ${USER}_test.leader_board_teams</li></ul></li><li><p>Run Leaderboard with Dataflow Runner</p><pre><code>python -m apache_beam.examples.complete.game.leader_board \
+--project=${YOUR_PROJECT} \
--region=${GCE_REGION} \
---topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \
---dataset ${USER}_test \
---runner DataflowRunner \
---temp_location=${YOUR_GS_BUCKET}/temp/ \
+--topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \
+--dataset ${USER}_test \
+--runner DataflowRunner \
+--temp_location=${YOUR_GS_BUCKET}/temp/ \
--sdk_location dist/*
</code></pre><p>Inspect results:</p><ul><li>Goto your Dataflow job console and check whether there is any error.</li><li>Goto your BigQuery console and check whether your ${USER}_test has leader_board_users and leader_board_teams table.</li><li>bq head -n 10 ${USER}_test.leader_board_users</li><li>bq head -n 10 ${USER}_test.leader_board_teams</li></ul></li><li><p>Run GameStats with Direct Runner</p><pre><code>python -m apache_beam.examples.complete.game.game_stats \
--project=${YOUR_PROJECT} \
--topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \
--dataset ${USER}_test \
--fixed_window_duration ${SOME_SMALL_DURATION}
-</code></pre><p>Inspect results:</p><ul><li>Check whether there is any error messages in console.</li><li>Goto your BigQuery console and check whether your ${USER}_test has game_stats_teams and game_stats_sessions table.</li><li>bq head -n 10 ${USER}_test.game_stats_teams</li><li>bq head -n 10 ${USER}_test.game_stats_sessions</li></ul></li><li><p>Run GameStats with Dataflow Runner</p><pre><code>python -m apache_beam.examples.complete.game.game_stats \
+</code></pre><p>Inspect results:</p><ul><li>Check whether there is any error messages in console.</li><li>Goto your BigQuery console and check whether your ${USER}_test has game_stats_teams and game_stats_sessions table.</li><li>bq head -n 10 ${USER}_test.game_stats_teams</li><li>bq head -n 10 ${USER}_test.game_stats_sessions</li></ul></li><li><p>Run GameStats with Dataflow Runner</p><pre><code>python -m apache_beam.examples.complete.game.game_stats \
--project=${YOUR_PROJECT} \
--region=${GCE_REGION} \
---topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \
---dataset ${USER}_test \
---runner DataflowRunner \
---temp_location=${YOUR_GS_BUCKET}/temp/ \
+--topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \
+--dataset ${USER}_test \
+--runner DataflowRunner \
+--temp_location=${YOUR_GS_BUCKET}/temp/ \
--sdk_location dist/* \
--fixed_window_duration ${SOME_SMALL_DURATION}
</code></pre><p>Inspect results:</p><ul><li>Goto your Dataflow job console and check whether there is any error.</li><li>Goto your BigQuery console and check whether your ${USER}_test has game_stats_teams and game_stats_sessions table.</li><li>bq head -n 10 ${USER}_test.game_stats_teams</li><li>bq head -n 10 ${USER}_test.game_stats_sessions</li></ul></li></ul></li></ul><h3 id=fix-any-issues>Fix any issues</h3><p>Any issues identified during the community review and vote should be fixed i [...]
diff --git a/website/generated-content/documentation/dsls/sql/extensions/user-defined-functions/index.html b/website/generated-content/documentation/dsls/sql/extensions/user-defined-functions/index.html
index 14c3cd1..e0e3254 100644
--- a/website/generated-content/documentation/dsls/sql/extensions/user-defined-functions/index.html
+++ b/website/generated-content/documentation/dsls/sql/extensions/user-defined-functions/index.html
@@ -24,14 +24,14 @@ returns one scalar value.</li><li>A <code>SerializableFunction</code>.</li></ul>
<span class=o>}</span>
<span class=c1>// Define a SQL query which calls the above UDFs
-</span><span class=c1></span><span class=n>String</span> <span class=n>sql</span> <span class=o>=</span>
+</span><span class=c1></span><span class=n>String</span> <span class=n>sql</span> <span class=o>=</span>
<span class=s>"SELECT f_int, cubic1(f_int), cubic2(f_int)"</span>
<span class=o>+</span> <span class=s>"FROM PCOLLECTION "</span>
<span class=o>+</span> <span class=s>"WHERE f_int = 2"</span><span class=o>;</span>
<span class=c1>// Create and apply the PTransform representing the query.
-</span><span class=c1>// Register the UDFs used in the query by calling '.registerUdf()' with
-</span><span class=c1>// either a class which implements BeamSqlUdf or with
+</span><span class=c1>// Register the UDFs used in the query by calling '.registerUdf()' with
+</span><span class=c1>// either a class which implements BeamSqlUdf or with
</span><span class=c1>// an instance of the SerializableFunction;
</span><span class=c1></span><span class=n>PCollection</span><span class=o><</span><span class=n>Row</span><span class=o>></span> <span class=n>result</span> <span class=o>=</span>
<span class=n>input</span><span class=o>.</span><span class=na>apply</span><span class=o>(</span>
@@ -71,13 +71,13 @@ example above:</p><div class=language-java><div class=highlight><pre class=chrom
<span class=o>}</span>
<span class=c1>// Define a SQL query which calls the above UDAF
-</span><span class=c1></span><span class=n>String</span> <span class=n>sql</span> <span class=o>=</span>
+</span><span class=c1></span><span class=n>String</span> <span class=n>sql</span> <span class=o>=</span>
<span class=s>"SELECT f_int1, squaresum(f_int2) "</span>
<span class=o>+</span> <span class=s>"FROM PCOLLECTION "</span>
<span class=o>+</span> <span class=s>"GROUP BY f_int2"</span><span class=o>;</span>
-
+
<span class=c1>// Create and apply the PTransform representing the query.
-</span><span class=c1>// Register the UDAFs used in the query by calling '.registerUdaf()' by
+</span><span class=c1>// Register the UDAFs used in the query by calling '.registerUdaf()' by
</span><span class=c1>// providing it an instance of the CombineFn
</span><span class=c1></span><span class=n>PCollection</span><span class=o><</span><span class=n>Row</span><span class=o>></span> <span class=n>result</span> <span class=o>=</span>
<span class=n>input</span><span class=o>.</span><span class=na>apply</span><span class=o>(</span>
diff --git a/website/generated-content/documentation/dsls/sql/extensions/windowing-and-triggering/index.html b/website/generated-content/documentation/dsls/sql/extensions/windowing-and-triggering/index.html
index 51ad721..091751f 100644
--- a/website/generated-content/documentation/dsls/sql/extensions/windowing-and-triggering/index.html
+++ b/website/generated-content/documentation/dsls/sql/extensions/windowing-and-triggering/index.html
@@ -6,20 +6,20 @@ to a <code>BeamSql</code> transform</li><li>you can use windowing extensions in
the windowing of your input <code>PCollections</code></li></ul><p>Triggering can only be used by setting it on your input <code>PCollections</code>; there
are no SQL extensions for specifying triggering.</p><p>This section covers the use of SQL extensions to directly apply windowing.</p><p>Beam SQL supports windowing functions specified in <code>GROUP BY</code> clause.
<code>TIMESTAMP</code> field is required in this case. It is used as event timestamp for
-rows.</p><p>Supported windowing functions:</p><ul><li><code>TUMBLE</code>, or fixed windows. Example of how define a fixed window with duration of 1 hour:</li></ul><pre><code> SELECT f_int, COUNT(*)
- FROM PCOLLECTION
- GROUP BY
+rows.</p><p>Supported windowing functions:</p><ul><li><code>TUMBLE</code>, or fixed windows. Example of how define a fixed window with duration of 1 hour:</li></ul><pre><code> SELECT f_int, COUNT(*)
+ FROM PCOLLECTION
+ GROUP BY
f_int,
TUMBLE(f_timestamp, INTERVAL '1' HOUR)
</code></pre><ul><li><code>HOP</code>, or sliding windows. Example of how to define a sliding windows for every 30 minutes with 1 hour duration:</li></ul><pre><code> SELECT f_int, COUNT(*)
- FROM PCOLLECTION
- GROUP BY
- f_int,
+ FROM PCOLLECTION
+ GROUP BY
+ f_int,
HOP(f_timestamp, INTERVAL '30' MINUTE, INTERVAL '1' HOUR)
-</code></pre><ul><li><code>SESSION</code>, session windows. Example of how to define a session window with 5 minutes gap duration:</li></ul><pre><code> SELECT f_int, COUNT(*)
- FROM PCOLLECTION
- GROUP BY
- f_int,
+</code></pre><ul><li><code>SESSION</code>, session windows. Example of how to define a session window with 5 minutes gap duration:</li></ul><pre><code> SELECT f_int, COUNT(*)
+ FROM PCOLLECTION
+ GROUP BY
+ f_int,
SESSION(f_timestamp, INTERVAL '5' MINUTE)
</code></pre><p><strong>Note:</strong> When no windowing function is specified in the query, then windowing strategy of the input <code>PCollections</code> is unchanged by the SQL query. If windowing function is specified in the query, then the windowing function of the <code>PCollection</code> is updated accordingly, but trigger stays unchanged.</p></div></div><footer class=footer><div class=footer__contained><div class=footer__cols><div class=footer__cols__col><div class=footer__cols__ [...]
<a href=http://www.apache.org>The Apache Software Foundation</a>
diff --git a/website/generated-content/documentation/dsls/sql/shell/index.html b/website/generated-content/documentation/dsls/sql/shell/index.html
index c6989ba..7128c0b 100644
--- a/website/generated-content/documentation/dsls/sql/shell/index.html
+++ b/website/generated-content/documentation/dsls/sql/shell/index.html
@@ -5,7 +5,7 @@
./sdks/java/extensions/sql/shell/build/install/shell/bin/shell
</code></pre><p>After you run the commands, the SQL shell starts and you can type queries:</p><pre><code>Welcome to Beam SQL 2.6.0-SNAPSHOT (based on sqlline version 1.4.0)
-0: BeamSQL>
+0: BeamSQL>
</code></pre><p><em>Note: If you haven’t built the project before running the Gradle command, the command will take a few minutes as Gradle must build all dependencies first.</em></p><p>The shell converts the queries into Beam pipelines, runs them using <code>DirectRunner</code>, and returns the results as tables when the pipelines finish:</p><pre><code>0: BeamSQL> SELECT 'foo' AS NAME, 'bar' AS TYPE, 'num' AS NUMBER;
+------+------+--------+
| NAME | TYPE | NUMBER |
diff --git a/website/generated-content/documentation/dsls/sql/walkthrough/index.html b/website/generated-content/documentation/dsls/sql/walkthrough/index.html
index 3f3be2f..6f02f48 100644
--- a/website/generated-content/documentation/dsls/sql/walkthrough/index.html
+++ b/website/generated-content/documentation/dsls/sql/walkthrough/index.html
@@ -8,7 +8,7 @@ by its associated <a href=https://beam.apache.org/releases/javadoc/2.23.0/index.
You can use the <a href=https://beam.apache.org/releases/javadoc/2.23.0/index.html?org/apache/beam/sdk/schemas/Schema.html>Schema.builder()</a> to create
<code>Schemas</code>. See <a href=/documentation/dsls/sql/data-types>Data
Types</a> for more details on supported primitive data types.</p><p>A <code>PCollection<Row></code> can be obtained multiple ways, for example:</p><ul><li><p><strong>From in-memory data</strong> (typically for unit testing).</p><p><strong>Note:</strong> you have to explicitly specify the <code>Row</code> coder. In this example we’re doing it by calling <code>Create.of(..)</code>:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-la [...]
-</span><span class=c1></span> <span class=n>Schema</span> <span class=n>appSchema</span> <span class=o>=</span>
+</span><span class=c1></span> <span class=n>Schema</span> <span class=n>appSchema</span> <span class=o>=</span>
<span class=n>Schema</span>
<span class=o>.</span><span class=na>builder</span><span class=o>()</span>
<span class=o>.</span><span class=na>addInt32Field</span><span class=o>(</span><span class=s>"appId"</span><span class=o>)</span>
@@ -17,14 +17,14 @@ Types</a> for more details on supported primitive data types.</p><p>A <code>PCol
<span class=o>.</span><span class=na>build</span><span class=o>();</span>
<span class=c1>// Create a concrete row with that type.
-</span><span class=c1></span> <span class=n>Row</span> <span class=n>row</span> <span class=o>=</span>
+</span><span class=c1></span> <span class=n>Row</span> <span class=n>row</span> <span class=o>=</span>
<span class=n>Row</span>
<span class=o>.</span><span class=na>withSchema</span><span class=o>(</span><span class=n>appSchema</span><span class=o>)</span>
<span class=o>.</span><span class=na>addValues</span><span class=o>(</span><span class=n>1</span><span class=o>,</span> <span class=s>"Some cool app"</span><span class=o>,</span> <span class=k>new</span> <span class=n>Date</span><span class=o>())</span>
<span class=o>.</span><span class=na>build</span><span class=o>();</span>
<span class=c1>// Create a source PCollection containing only that row
-</span><span class=c1></span> <span class=n>PCollection</span><span class=o><</span><span class=n>Row</span><span class=o>></span> <span class=n>testApps</span> <span class=o>=</span>
+</span><span class=c1></span> <span class=n>PCollection</span><span class=o><</span><span class=n>Row</span><span class=o>></span> <span class=n>testApps</span> <span class=o>=</span>
<span class=n>PBegin</span>
<span class=o>.</span><span class=na>in</span><span class=o>(</span><span class=n>p</span><span class=o>)</span>
<span class=o>.</span><span class=na>apply</span><span class=o>(</span><span class=n>Create</span>
@@ -49,14 +49,14 @@ Types</a> for more details on supported primitive data types.</p><p>A <code>PCol
<span class=c1>// Get the current POJO instance
</span><span class=c1></span> <span class=n>AppPojo</span> <span class=n>pojo</span> <span class=o>=</span> <span class=n>c</span><span class=o>.</span><span class=na>element</span><span class=o>();</span>
- <span class=c1>// Create a Row with the appSchema schema
+ <span class=c1>// Create a Row with the appSchema schema
</span><span class=c1></span> <span class=c1>// and values from the current POJO
-</span><span class=c1></span> <span class=n>Row</span> <span class=n>appRow</span> <span class=o>=</span>
+</span><span class=c1></span> <span class=n>Row</span> <span class=n>appRow</span> <span class=o>=</span>
<span class=n>Row</span>
<span class=o>.</span><span class=na>withSchema</span><span class=o>(</span><span class=n>appSchema</span><span class=o>)</span>
<span class=o>.</span><span class=na>addValues</span><span class=o>(</span>
- <span class=n>pojo</span><span class=o>.</span><span class=na>appId</span><span class=o>,</span>
- <span class=n>pojo</span><span class=o>.</span><span class=na>description</span><span class=o>,</span>
+ <span class=n>pojo</span><span class=o>.</span><span class=na>appId</span><span class=o>,</span>
+ <span class=n>pojo</span><span class=o>.</span><span class=na>description</span><span class=o>,</span>
<span class=n>pojo</span><span class=o>.</span><span class=na>timestamp</span><span class=o>)</span>
<span class=o>.</span><span class=na>build</span><span class=o>();</span>
@@ -72,8 +72,8 @@ to either a single <code>PCollection</code> or a <code>PCollectionTuple</code> w
<span class=s>"SELECT appId, description, rowtime "</span>
<span class=o>+</span> <span class=s>"FROM PCOLLECTION "</span>
<span class=o>+</span> <span class=s>"WHERE id=1"</span><span class=o>));</span>
- </code></pre></div></div></p></li><li><p>when applying to a <code>PCollectionTuple</code>, the tuple tag for each <code>PCollection</code> in the tuple defines the table name that may be used to query it. Note that table names are bound to the specific <code>PCollectionTuple</code>, and thus are only valid in the context of queries applied to it.</p><p>For example, you can join two <code>PCollections</code>:<br><div class=language-java><div class=highlight><pre class=chroma><code cla [...]
-</span><span class=c1></span> <span class=n>Schema</span> <span class=n>reviewSchema</span> <span class=o>=</span>
+ </code></pre></div></div></p></li><li><p>when applying to a <code>PCollectionTuple</code>, the tuple tag for each <code>PCollection</code> in the tuple defines the table name that may be used to query it. Note that table names are bound to the specific <code>PCollectionTuple</code>, and thus are only valid in the context of queries applied to it.</p><p>For example, you can join two <code>PCollections</code>:<div class=language-java><div class=highlight><pre class=chroma><code class=l [...]
+</span><span class=c1></span> <span class=n>Schema</span> <span class=n>reviewSchema</span> <span class=o>=</span>
<span class=n>Schema</span>
<span class=o>.</span><span class=na>builder</span><span class=o>()</span>
<span class=o>.</span><span class=na>addInt32Field</span><span class=o>(</span><span class=s>"appId"</span><span class=o>)</span>
@@ -81,7 +81,7 @@ to either a single <code>PCollection</code> or a <code>PCollectionTuple</code> w
<span class=o>.</span><span class=na>addFloatField</span><span class=o>(</span><span class=s>"rating"</span><span class=o>)</span>
<span class=o>.</span><span class=na>addDateTimeField</span><span class=o>(</span><span class=s>"rowtime"</span><span class=o>)</span>
<span class=o>.</span><span class=na>build</span><span class=o>();</span>
-
+
<span class=c1>// Obtain the reviews records with this schema
</span><span class=c1></span> <span class=n>PCollection</span><span class=o><</span><span class=n>Row</span><span class=o>></span> <span class=n>reviewsRows</span> <span class=o>=</span> <span class=o>...</span>
@@ -91,8 +91,8 @@ to either a single <code>PCollection</code> or a <code>PCollectionTuple</code> w
<span class=k>new</span> <span class=n>TupleTag</span><span class=o><>(</span><span class=s>"Apps"</span><span class=o>),</span> <span class=n>appsRows</span><span class=o>),</span> <span class=c1>// appsRows from the previous example
</span><span class=c1></span> <span class=k>new</span> <span class=n>TupleTag</span><span class=o><>(</span><span class=s>"Reviews"</span><span class=o>),</span> <span class=n>reviewsRows</span><span class=o>));</span>
- <span class=c1>// Compute the total number of reviews
-</span><span class=c1></span> <span class=c1>// and average rating per app
+ <span class=c1>// Compute the total number of reviews
+</span><span class=c1></span> <span class=c1>// and average rating per app
</span><span class=c1></span> <span class=c1>// by joining two PCollections
</span><span class=c1></span> <span class=n>PCollection</span><span class=o><</span><span class=n>Row</span><span class=o>></span> <span class=n>output</span> <span class=o>=</span> <span class=n>namesAndFoods</span><span class=o>.</span><span class=na>apply</span><span class=o>(</span>
<span class=n>SqlTransform</span><span class=o>.</span><span class=na>query</span><span class=o>(</span>
diff --git a/website/generated-content/documentation/index.xml b/website/generated-content/documentation/index.xml
index 2f9cf4c..8acc680 100644
--- a/website/generated-content/documentation/index.xml
+++ b/website/generated-content/documentation/index.xml
@@ -1163,7 +1163,7 @@ There are scenarios when this may prove faster than accessing content through th
<span class="s">&#34;my_snaphshot&#34;</span><span class="o">,</span>
<span class="n">TableName</span><span class="o">.</span><span class="na">valueOf</span><span class="o">(</span><span class="s">&#34;my_table&#34;</span><span class="o">),</span>
<span class="n">HBaseProtos</span><span class="o">.</span><span class="na">SnapshotDescription</span><span class="o">.</span><span class="na">Type</span><span class="o">.</span><span class="na">FLUSH</span><span class="o">);</span>
-<span class="o">}</span> </code></pre></div>
+<span class="o">}</span></code></pre></div>
</div>
</p>
<div class=language-py>
@@ -3534,8 +3534,7 @@ To access the timestamp of an input element, add a keyword parameter default to
<span class="k">class</span> <span class="nc">ProcessRecord</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">element</span><span class="p">,</span> <span class="n">timestamp</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="o">.</span><span class="n">TimestampParam</span><span class="p">):&l [...]
<span class="c1"># access timestamp of element.</span>
-<span class="k">pass</span>
-</code></pre></div>
+<span class="k">pass</span></code></pre></div>
</div>
<p class="language-java"><strong>Window:</strong>
To access the window an input element falls into, add a parameter of the type of the window used for the input <code>PCollection</code>.
@@ -3557,8 +3556,7 @@ If an element falls in multiple windows (for example, this will happen when usin
<span class="k">class</span> <span class="nc">ProcessRecord</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">element</span><span class="p">,</span> <span class="n">window</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="o">.</span><span class="n">WindowParam</span><span class="p">):</span>
<span class="c1"># access window e.g. window.end.micros</span>
-<span class="k">pass</span>
-</code></pre></div>
+<span class="k">pass</span></code></pre></div>
</div>
<p class="language-java"><strong>PaneInfo:</strong>
When triggers are used, Beam provides a <code>PaneInfo</code> object that contains information about the current firing. Using <code>PaneInfo</code>
@@ -3577,8 +3575,7 @@ This feature implementation in Python SDK is not fully completed; see more at &l
<span class="k">class</span> <span class="nc">ProcessRecord</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">element</span><span class="p">,</span> <span class="n">pane_info</span><span class="o">=</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="o">.</span><span class="n">PaneInfoParam</span><span class="p">):</span>
<span class="c1"># access pane info, e.g. pane_info.is_first, pane_info.is_last, pane_info.timing</span>
-<span class="k">pass</span>
-</code></pre></div>
+<span class="k">pass</span></code></pre></div>
</div>
<p class="language-java"><strong>PipelineOptions:</strong>
The <code>PipelineOptions</code> for the current pipeline can always be accessed in a process method by adding it
@@ -3951,7 +3948,7 @@ Schema rows; Beam uses a special coder to encode schema types.</p>
<p>While schemas themselves are language independent, they are designed to embed naturally into the programming languages
of the Beam SDK being used. This allows Beam users to continue using native types while reaping the advantage of
having Beam understand their element schemas.</p>
-<p class="language-java">In Java you could use the following set of classes to represent the purchase schema. Beam will automatically<br>
+<p class="language-java">In Java you could use the following set of classes to represent the purchase schema. Beam will automatically
infer the correct schema based on the members of the class.</p>
<div class=language-java>
<div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"><span class="nd">@DefaultSchema</span><span class="o">(</span><span class="n">JavaBeanSchema</span><span class="o">.</span><span class="na">class</span><span class="o">)</span>
@@ -4231,7 +4228,7 @@ exclude these fields. Note that ignored fields will not be included in the encod
not owned by the Beam pipeline author. In these cases the schema inference can be triggered programmatically in
pipeline’s main function as follows:</p>
<div class=language-java>
-<div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"> <span class="n">pipeline</span><span class="o">.</span><span class="na">getSchemaRegistry</span><span class="o">().</span><span class="na">registerPOJO</span><span class="o">(</span><span class="n">TransactionPOJO</span><span class="o">.</span><span class="na">class</span><span class="o">);</span> </code></pre></div>
+<div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"> <span class="n">pipeline</span><span class="o">.</span><span class="na">getSchemaRegistry</span><span class="o">().</span><span class="na">registerPOJO</span><span class="o">(</span><span class="n">TransactionPOJO</span><span class="o">.</span><span class="na">class</span><span class="o">);</span></code></pre></div>
</div>
<h5 id="java-beans"><strong>Java Beans</strong></h5>
<p>Java Beans are a de-facto standard for creating reusable property classes in Java. While the full
@@ -5515,8 +5512,7 @@ windowing function:</p>
<span class="n">FixedWindows</span><span class="p">(</span><span class="mi">60</span><span class="p">),</span>
<span class="n">trigger</span><span class="o">=</span><span class="n">AfterProcessingTime</span><span class="p">(</span><span class="mi">60</span><span class="p">),</span>
<span class="n">allowed_lateness</span><span class="o">=</span><span class="mi">1800</span><span class="p">)</span> <span class="c1"># 30 minutes</span>
-<span class="o">|</span> <span class="o">...</span>
-</code></pre></div>
+<span class="o">|</span> <span class="o">...</span></code></pre></div>
</div>
<p>This allowed lateness propagates to all <code>PCollection</code>s derived as a result of
applying transforms to the original <code>PCollection</code>. If you want to change the
@@ -6003,7 +5999,7 @@ id, and timers in different timer families are independent.</p>
</div>
<h4 id="timer-output-timestamps">11.3.4 Timer output timestamps</h4>
<p>By default, event-time timers will hold the output watermark of the <code>ParDo</code> to the timestamp of the timer. This means
-that if a timer is set to 12pm, any windowed aggregations or event-time timers later in the pipeline graph that finish<br>
+that if a timer is set to 12pm, any windowed aggregations or event-time timers later in the pipeline graph that finish
after 12pm will not expire. The timestamp of the timer is also the default output timestamp for the timer callback. This
means that any elements output from the onTimer method will have a timestamp equal to the timestamp of the timer firing.
For processing-time timers, the default output timestamp and watermark hold is the value of the input watermark at the
@@ -6178,7 +6174,7 @@ This can be done by updating a timer that garbage collects state. For example<
<span class="n">timer</span> <span class="o">=</span> <span class="n">DoFn</span><span class="o">.</span><span class="n">TimerParam</span><span class="p">(</span><span class="n">TIMER</span><span class="p">)):</span>
<span class="n">update_state</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="n">element</span><span class="p">)</span>
<span class="n">max_timestamp</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">micros</span><span class="p">)</span>
-<span class="c1"># Set the timer to be one hour after the maximum timestamp seen. This will keep overwriting the same timer, so </span>
+<span class="c1"># Set the timer to be one hour after the maximum timestamp seen. This will keep overwriting the same timer, so</span>
<span class="c1"># as long as there is activity on this key the state will stay active. Once the key goes inactive for one hour&#39;s</span>
<span class="c1"># worth of event time (as measured by the watermark), then the gc timer will fire.</span>
<span class="n">expiration_time</span> <span class="o">=</span> <span class="n">Timestamp</span><span class="p">(</span><span class="n">micros</span><span class="o">=</span><span class="n">max_timestamp</span><span class="o">.</span><span class="n">read</span><span class="p">())</span> <span class="o">+</span> <span class="n">Duration</span><span class="p">(</span><span class="n">seconds</span><span cl [...]
diff --git a/website/generated-content/documentation/io/built-in/hadoop/index.html b/website/generated-content/documentation/io/built-in/hadoop/index.html
index cfc13fb..8d0fa4d 100644
--- a/website/generated-content/documentation/io/built-in/hadoop/index.html
+++ b/website/generated-content/documentation/io/built-in/hadoop/index.html
@@ -93,7 +93,7 @@ There are scenarios when this may prove faster than accessing content through th
<span class=s>"my_snaphshot"</span><span class=o>,</span>
<span class=n>TableName</span><span class=o>.</span><span class=na>valueOf</span><span class=o>(</span><span class=s>"my_table"</span><span class=o>),</span>
<span class=n>HBaseProtos</span><span class=o>.</span><span class=na>SnapshotDescription</span><span class=o>.</span><span class=na>Type</span><span class=o>.</span><span class=na>FLUSH</span><span class=o>);</span>
-<span class=o>}</span> </code></pre></div></div></p><div class=language-py><div class=highlight><pre class=chroma><code class=language-py data-lang=py> <span class=c1># The Beam SDK for Python does not support Hadoop Input/Output Format IO.</span></code></pre></div></div><p>A <code>TableSnapshotInputFormat</code> is configured as follows:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=c1>// Construct a typical HBase scan
+<span class=o>}</span></code></pre></div></div></p><div class=language-py><div class=highlight><pre class=chroma><code class=language-py data-lang=py> <span class=c1># The Beam SDK for Python does not support Hadoop Input/Output Format IO.</span></code></pre></div></div><p>A <code>TableSnapshotInputFormat</code> is configured as follows:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=c1>// Construct a typical HBase scan
</span><span class=c1></span><span class=n>Scan</span> <span class=n>scan</span> <span class=o>=</span> <span class=k>new</span> <span class=n>Scan</span><span class=o>();</span>
<span class=n>scan</span><span class=o>.</span><span class=na>setCaching</span><span class=o>(</span><span class=n>1000</span><span class=o>);</span>
<span class=n>scan</span><span class=o>.</span><span class=na>setBatch</span><span class=o>(</span><span class=n>1000</span><span class=o>);</span>
diff --git a/website/generated-content/documentation/io/built-in/hcatalog/index.html b/website/generated-content/documentation/io/built-in/hcatalog/index.html
index e9b3a66..a13a0ce 100644
--- a/website/generated-content/documentation/io/built-in/hcatalog/index.html
+++ b/website/generated-content/documentation/io/built-in/hcatalog/index.html
@@ -2,7 +2,7 @@
<span class=sr-only>Toggle navigation</span>
<span class=icon-bar></span><span class=icon-bar></span><span class=icon-bar></span></button>
<a href=/ class=navbar-brand><img alt=Brand style=height:25px src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask closed"></div><div id=navbar class="navbar-container closed"><ul class="nav navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a href=/documentation/>Documentation</a></li><li><a href=/documentation/sdks/java/>Languages</a></li><li><a href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a href=/roadmap/>Roadmap</a></li>< [...]
-<span class=n>configProperties</span><span class=o>.</span><span class=na>put</span><span class=o>(</span><span class=s>"hive.metastore.uris"</span><span class=o>,</span><span class=s>"thrift://metastore-host:port"</span><span class=o>);</span>
+<span class=n>configProperties</span><span class=o>.</span><span class=na>put</span><span class=o>(</span><span class=s>"hive.metastore.uris"</span><span class=o>,</span><span class=s>"thrift://metastore-host:port"</span><span class=o>);</span>
<span class=n>pipeline</span>
<span class=o>.</span><span class=na>apply</span><span class=o>(</span><span class=n>HCatalogIO</span><span class=o>.</span><span class=na>read</span><span class=o>()</span>
<span class=o>.</span><span class=na>withConfigProperties</span><span class=o>(</span><span class=n>configProperties</span><span class=o>)</span>
diff --git a/website/generated-content/documentation/io/built-in/snowflake/index.html b/website/generated-content/documentation/io/built-in/snowflake/index.html
index 314d4de..c7eab09 100644
--- a/website/generated-content/documentation/io/built-in/snowflake/index.html
+++ b/website/generated-content/documentation/io/built-in/snowflake/index.html
@@ -22,10 +22,10 @@
<span class=o>.</span><span class=na>withStorageIntegrationName</span><span class=o>(</span><span class=s>"STORAGE INTEGRATION NAME"</span><span class=o>)</span>
<span class=o>.</span><span class=na>withUserDataMapper</span><span class=o>(</span><span class=n>mapper</span><span class=o>)</span>
<span class=o>)</span></code></pre></div></div>Replace type with the data type of the PCollection object to write; for example, SnowflakeIO.<string> for an input PCollection of Strings.</p><p>All the below parameters are required:</p><ul><li><p><code>.withDataSourceConfiguration()</code> Accepts a DatasourceConfiguration object.</p></li><li><p><code>.to()</code> Accepts the target Snowflake table name.</p></li><li><p><code>.withStagingBucketName()</code> Accepts a cloud bucket path ended [...]
--Example: <code>.withStagingBucketName("gs://mybucket/my/dir/")</code></p></li><li><p><code>.withStorageIntegrationName()</code> Accepts a name of a Snowflake storage integration object created according to Snowflake documentationt. Example:<pre><code>CREATE OR REPLACE STORAGE INTEGRATION test_integration
-TYPE = EXTERNAL_STAGE
-STORAGE_PROVIDER = GCS
-ENABLED = TRUE
+-Example: <code>.withStagingBucketName("gs://mybucket/my/dir/")</code></p></li><li><p><code>.withStorageIntegrationName()</code> Accepts a name of a Snowflake storage integration object created according to Snowflake documentationt. Example:<pre><code>CREATE OR REPLACE STORAGE INTEGRATION test_integration
+TYPE = EXTERNAL_STAGE
+STORAGE_PROVIDER = GCS
+ENABLED = TRUE
STORAGE_ALLOWED_LOCATIONS = ('gcs://bucket/');</code></pre>Then:<pre><code>.withStorageIntegrationName(test_integration)</code></pre></p></li><li><p><code>.withUserDataMapper()</code> Accepts the UserDataMapper function that will map a user’s PCollection to an array of String values <code>(String[])</code>.</p></li></ul><p><strong>Note</strong>:
SnowflakeIO uses COPY statements behind the scenes to write (using <a href=https://docs.snowflake.net/manuals/sql-reference/sql/copy-into-table.html>COPY to table</a>). StagingBucketName will be used to save CSV files which will end up in Snowflake. Those CSV files will be saved under the “stagingBucketName” path.</p><h3 id=userdatamapper-function>UserDataMapper function</h3><p>The UserDataMapper function is required to map data from a PCollection to an array of String values before the [...]
<span class=k>return</span> <span class=o>(</span><span class=n>SnowflakeIO</span><span class=o>.</span><span class=na>UserDataMapper</span><span class=o><</span><span class=n>Long</span><span class=o>>)</span> <span class=n>recordLine</span> <span class=o>-></span> <span class=k>new</span> <span class=n>String</span><span class=o>[]</span> <span class=o>{</span><span class=n>recordLine</span><span class=o>.</span><span class=na>toString</span><span class=o>()};</span>
@@ -77,10 +77,10 @@ A table schema is a list of <code>SFColumn</code> objects with name and type cor
<span class=o>.</span><span class=na>withStorageIntegrationName</span><span class=o>(</span><span class=s>"STORAGE INTEGRATION NAME"</span><span class=o>)</span>
<span class=o>.</span><span class=na>withCsvMapper</span><span class=o>(</span><span class=n>mapper</span><span class=o>)</span>
<span class=o>.</span><span class=na>withCoder</span><span class=o>(</span><span class=n>coder</span><span class=o>));</span>
-<span class=o>)</span></code></pre></div></div>Where all below parameters are required:</p><ul><li><p><code>.withDataSourceConfiguration(...)</code></p><ul><li>Accepts a DataSourceConfiguration object.</li></ul></li><li><p><code>.fromTable(...) or .fromQuery(...)</code></p><ul><li>Specifies a Snowflake table name or custom SQL query.</li></ul></li><li><p><code>.withStagingBucketName()</code></p><ul><li>Accepts a cloud bucket name.</li></ul></li><li><p><code>.withStorageIntegrationName()< [...]
-TYPE = EXTERNAL_STAGE
-STORAGE_PROVIDER = GCS
-ENABLED = TRUE
+<span class=o>)</span></code></pre></div></div>Where all below parameters are required:</p><ul><li><p><code>.withDataSourceConfiguration(...)</code></p><ul><li>Accepts a DataSourceConfiguration object.</li></ul></li><li><p><code>.fromTable(...) or .fromQuery(...)</code></p><ul><li>Specifies a Snowflake table name or custom SQL query.</li></ul></li><li><p><code>.withStagingBucketName()</code></p><ul><li>Accepts a cloud bucket name.</li></ul></li><li><p><code>.withStorageIntegrationName()< [...]
+TYPE = EXTERNAL_STAGE
+STORAGE_PROVIDER = GCS
+ENABLED = TRUE
STORAGE_ALLOWED_LOCATIONS = ('gcs://bucket/');</code></pre>Then:<pre><code>.withStorageIntegrationName(test_integration)</code></pre></p></li><li><p><code>.withCsvMapper(mapper)</code></p><ul><li>Accepts a <a href=https://beam.apache.org/documentation/io/built-in/snowflake/#csvmapper>CSVMapper</a> instance for mapping String[] to USER_DATA_TYPE.</li></ul></li><li><p><code>.withCoder(coder)</code></p><ul><li>Accepts the <a href=https://beam.apache.org/releases/javadoc/2.0.0/org/ap [...]
SnowflakeIO uses COPY statements behind the scenes to read (using <a href=https://docs.snowflake.net/manuals/sql-reference/sql/copy-into-location.html>COPY to location</a>) files staged in cloud storage.StagingBucketName will be used as a temporary location for storing CSV files. Those temporary directories will be named <code>sf_copy_csv_DATE_TIME_RANDOMSUFFIX</code> and they will be removed automatically once Read operation finishes.</p><h3 id=csvmapper>CSVMapper</h3><p>SnowflakeIO use [...]
<span class=k>return</span> <span class=o>(</span><span class=n>SnowflakeIO</span><span class=o>.</span><span class=na>CsvMapper</span><span class=o><</span><span class=n>GenericRecord</span><span class=o>>)</span>
diff --git a/website/generated-content/documentation/io/testing/index.html b/website/generated-content/documentation/io/testing/index.html
index 3488a17..9fe0ff2 100644
--- a/website/generated-content/documentation/io/testing/index.html
+++ b/website/generated-content/documentation/io/testing/index.html
@@ -2,7 +2,7 @@
<span class=sr-only>Toggle navigation</span>
<span class=icon-bar></span><span class=icon-bar></span><span class=icon-bar></span></button>
<a href=/ class=navbar-brand><img alt=Brand style=height:25px src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask closed"></div><div id=navbar class="navbar-container closed"><ul class="nav navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a href=/documentation/>Documentation</a></li><li><a href=/documentation/sdks/java/>Languages</a></li><li><a href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a href=/roadmap/>Roadmap</a></li>< [...]
-</code></pre><p>Example usage on HDFS filesystem and Direct runner:</p><p>NOTE: Below setup will only work when /etc/hosts file contains entries with hadoop namenode and hadoop datanodes external IPs. Please see explanation in: <a href=https://github.com/apache/beam/blob/master/.test-infra/kubernetes/hadoop/SmallITCluster/pkb-config.yml>Small Cluster config file</a> and <a href=https://github.com/apache/beam/blob/master/.test-infra/kubernetes/hadoop/LargeITCluster/pkb-config.yml>Large Cl [...]
+</code></pre><p>Example usage on HDFS filesystem and Direct runner:</p><p>NOTE: Below setup will only work when /etc/hosts file contains entries with hadoop namenode and hadoop datanodes external IPs. Please see explanation in: <a href=https://github.com/apache/beam/blob/master/.test-infra/kubernetes/hadoop/SmallITCluster/pkb-config.yml>Small Cluster config file</a> and <a href=https://github.com/apache/beam/blob/master/.test-infra/kubernetes/hadoop/LargeITCluster/pkb-config.yml>Large Cl [...]
./gradlew integrationTest -p sdks/java/io/file-based-io-tests -DintegrationTestPipelineOptions='["--numberOfRecords=1000", "--filenamePrefix=hdfs://HDFS_NAMENODE:9000/XMLIOIT", "--hdfsConfiguration=[{\"fs.defaultFS\":\"hdfs://HDFS_NAMENODE:9000\",\"dfs.replication\":1,\"dfs.client.use.datanode.hostname\":\"true\" }]" ]' -DintegrationTestRunner=direct -Dfilesystem=hdfs --tests org.apache.beam.sdk.io.xml.XmlIOIT
</code></pre><p>Parameter descriptions:</p><table class=table><thead><tr><td><strong>Option</strong></td><td><strong>Function</strong></td></tr></thead><tbody><tr><td>-p sdks/java/io/file-based-io-tests/</td><td>Specifies the project submodule of the I/O to test.</td></tr><tr><td>-DintegrationTestPipelineOptions</td><td>Passes pipeline options directly to the test being run.</td></tr><tr><td>-DintegrationTestRunner</td><td>Runner to be used for running the test. Currently possible option [...]
diff --git a/website/generated-content/documentation/programming-guide/index.html b/website/generated-content/documentation/programming-guide/index.html
index e6b743e..44d6c8a 100644
--- a/website/generated-content/documentation/programming-guide/index.html
+++ b/website/generated-content/documentation/programming-guide/index.html
@@ -91,7 +91,7 @@ setter methods for each option, as in the following example for
adding <code>input</code> and <code>output</code> custom options:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=kd>public</span> <span class=kd>interface</span> <span class=nc>MyOptions</span> <span class=kd>extends</span> <span class=n>PipelineOptions</span> <span class=o>{</span>
<span class=n>String</span> <span class=nf>getInput</span><span class=o>();</span>
<span class=kt>void</span> <span class=nf>setInput</span><span class=o>(</span><span class=n>String</span> <span class=n>input</span><span class=o>);</span>
-
+
<span class=n>String</span> <span class=nf>getOutput</span><span class=o>();</span>
<span class=kt>void</span> <span class=nf>setOutput</span><span class=o>(</span><span class=n>String</span> <span class=n>output</span><span class=o>);</span>
<span class=o>}</span></code></pre></div></div><div class=language-py><div class=highlight><pre class=chroma><code class=language-py data-lang=py><span class=kn>from</span> <span class=nn>apache_beam.options.pipeline_options</span> <span class=kn>import</span> <span class=n>PipelineOptions</span>
@@ -1141,8 +1141,7 @@ To access the timestamp of an input element, add a keyword parameter default to
<span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=n>element</span><span class=p>,</span> <span class=n>timestamp</span><span class=o>=</span><span class=n>beam</span><span class=o>.</span><span class=n>DoFn</span><span class=o>.</span><span class=n>TimestampParam</span><span class=p>):</span>
<span class=c1># access timestamp of element.</span>
- <span class=k>pass</span>
- </code></pre></div></div><p class=language-java><strong>Window:</strong>
+ <span class=k>pass</span></code></pre></div></div><p class=language-java><strong>Window:</strong>
To access the window an input element falls into, add a parameter of the type of the window used for the input <code>PCollection</code>.
If the parameter is a window type (a subclass of <code>BoundedWindow</code>) that does not match the input <code>PCollection</code>, then an error
will be raised. If an element falls in multiple windows (for example, this will happen when using <code>SlidingWindows</code>), then the
@@ -1158,8 +1157,7 @@ If an element falls in multiple windows (for example, this will happen when usin
<span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=n>element</span><span class=p>,</span> <span class=n>window</span><span class=o>=</span><span class=n>beam</span><span class=o>.</span><span class=n>DoFn</span><span class=o>.</span><span class=n>WindowParam</span><span class=p>):</span>
<span class=c1># access window e.g. window.end.micros</span>
- <span class=k>pass</span>
- </code></pre></div></div><p class=language-java><strong>PaneInfo:</strong>
+ <span class=k>pass</span></code></pre></div></div><p class=language-java><strong>PaneInfo:</strong>
When triggers are used, Beam provides a <code>PaneInfo</code> object that contains information about the current firing. Using <code>PaneInfo</code>
you can determine whether this is an early or a late firing, and how many times this window has already fired for this key.</p><p class=language-py><strong>PaneInfo:</strong>
When triggers are used, Beam provides a <code>DoFn.PaneInfoParam</code> object that contains information about the current firing. Using <code>DoFn.PaneInfoParam</code>
@@ -1172,8 +1170,7 @@ This feature implementation in Python SDK is not fully completed; see more at <a
<span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=n>element</span><span class=p>,</span> <span class=n>pane_info</span><span class=o>=</span><span class=n>beam</span><span class=o>.</span><span class=n>DoFn</span><span class=o>.</span><span class=n>PaneInfoParam</span><span class=p>):</span>
<span class=c1># access pane info, e.g. pane_info.is_first, pane_info.is_last, pane_info.timing</span>
- <span class=k>pass</span>
- </code></pre></div></div><p class=language-java><strong>PipelineOptions:</strong>
+ <span class=k>pass</span></code></pre></div></div><p class=language-java><strong>PipelineOptions:</strong>
The <code>PipelineOptions</code> for the current pipeline can always be accessed in a process method by adding it
as a parameter:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=k>new</span> <span class=n>DoFn</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>String</span><span class=o>>()</span> <span class=o>{</span>
<span class=kd>public</span> <span class=kt>void</span> <span class=nf>processElement</span><span class=o>(</span><span class=nd>@Element</span> <span class=n>String</span> <span class=n>word</span><span class=o>,</span> <span class=n>PipelineOptions</span> <span class=n>options</span><span class=o>)</span> <span class=o>{</span>
@@ -1362,16 +1359,17 @@ and Beam will allow us to seamlessly convert between these types. Schemas also p
types across different programming-language APIs.</p><p>A <code>PCollection</code> with a schema does not need to have a <code>Coder</code> specified, as Beam knows how to encode and decode
Schema rows; Beam uses a special coder to encode schema types.</p><h3 id=schemas-for-pl-types>6.2. Schemas for programming language types</h3><p>While schemas themselves are language independent, they are designed to embed naturally into the programming languages
of the Beam SDK being used. This allows Beam users to continue using native types while reaping the advantage of
-having Beam understand their element schemas.</p><p class=language-java>In Java you could use the following set of classes to represent the purchase schema. Beam will automatically<br>infer the correct schema based on the members of the class.</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=nd>@DefaultSchema</span><span class=o>(</span><span class=n>JavaBeanSchema</span><span class=o>.</span><span class=na>class</spa [...]
+having Beam understand their element schemas.</p><p class=language-java>In Java you could use the following set of classes to represent the purchase schema. Beam will automatically
+infer the correct schema based on the members of the class.</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=nd>@DefaultSchema</span><span class=o>(</span><span class=n>JavaBeanSchema</span><span class=o>.</span><span class=na>class</span><span class=o>)</span>
<span class=kd>public</span> <span class=kd>class</span> <span class=nc>Purchase</span> <span class=o>{</span>
<span class=kd>public</span> <span class=n>String</span> <span class=nf>getUserId</span><span class=o>();</span> <span class=c1>// Returns the id of the user who made the purchase.
</span><span class=c1></span> <span class=kd>public</span> <span class=kt>long</span> <span class=nf>getItemId</span><span class=o>();</span> <span class=c1>// Returns the identifier of the item that was purchased.
</span><span class=c1></span> <span class=kd>public</span> <span class=n>ShippingAddress</span> <span class=nf>getShippingAddress</span><span class=o>();</span> <span class=c1>// Returns the shipping address, a nested type.
</span><span class=c1></span> <span class=kd>public</span> <span class=kt>long</span> <span class=nf>getCostCents</span><span class=o>();</span> <span class=c1>// Returns the cost of the item.
</span><span class=c1></span> <span class=kd>public</span> <span class=n>List</span><span class=o><</span><span class=n>Transaction</span><span class=o>></span> <span class=nf>getTransactions</span><span class=o>();</span> <span class=c1>// Returns the transactions that paid for this purchase (returns a list, since the purchase might be spread out over multiple credit cards).
-</span><span class=c1></span>
+</span><span class=c1></span>
<span class=nd>@SchemaCreate</span>
- <span class=kd>public</span> <span class=nf>Purchase</span><span class=o>(</span><span class=n>String</span> <span class=n>userId</span><span class=o>,</span> <span class=kt>long</span> <span class=n>itemId</span><span class=o>,</span> <span class=n>ShippingAddress</span> <span class=n>shippingAddress</span><span class=o>,</span> <span class=kt>long</span> <span class=n>costCents</span><span class=o>,</span>
+ <span class=kd>public</span> <span class=nf>Purchase</span><span class=o>(</span><span class=n>String</span> <span class=n>userId</span><span class=o>,</span> <span class=kt>long</span> <span class=n>itemId</span><span class=o>,</span> <span class=n>ShippingAddress</span> <span class=n>shippingAddress</span><span class=o>,</span> <span class=kt>long</span> <span class=n>costCents</span><span class=o>,</span>
<span class=n>List</span><span class=o><</span><span class=n>Transaction</span><span class=o>></span> <span class=n>transactions</span><span class=o>)</span> <span class=o>{</span>
<span class=o>...</span>
<span class=o>}</span>
@@ -1384,7 +1382,7 @@ having Beam understand their element schemas.</p><p class=language-java>In Java
<span class=nd>@Nullable</span> <span class=kd>public</span> <span class=n>String</span> <span class=nf>getState</span><span class=o>();</span>
<span class=kd>public</span> <span class=n>String</span> <span class=nf>getCountry</span><span class=o>();</span>
<span class=kd>public</span> <span class=n>String</span> <span class=nf>getPostCode</span><span class=o>();</span>
-
+
<span class=nd>@SchemaCreate</span>
<span class=kd>public</span> <span class=nf>ShippingAddress</span><span class=o>(</span><span class=n>String</span> <span class=n>streetAddress</span><span class=o>,</span> <span class=n>String</span> <span class=n>city</span><span class=o>,</span> <span class=nd>@Nullable</span> <span class=n>String</span> <span class=n>state</span><span class=o>,</span> <span class=n>String</span> <span class=n>country</span><span class=o>,</span>
<span class=n>String</span> <span class=n>postCode</span><span class=o>)</span> <span class=o>{</span>
@@ -1396,7 +1394,7 @@ having Beam understand their element schemas.</p><p class=language-java>In Java
<span class=kd>public</span> <span class=kd>class</span> <span class=nc>Transaction</span> <span class=o>{</span>
<span class=kd>public</span> <span class=n>String</span> <span class=nf>getBank</span><span class=o>();</span>
<span class=kd>public</span> <span class=kt>double</span> <span class=nf>getPurchaseAmount</span><span class=o>();</span>
-
+
<span class=nd>@SchemaCreate</span>
<span class=kd>public</span> <span class=nf>Transaction</span><span class=o>(</span><span class=n>String</span> <span class=n>bank</span><span class=o>,</span> <span class=kt>double</span> <span class=n>purchaseAmount</span><span class=o>)</span> <span class=o>{</span>
<span class=o>...</span>
@@ -1446,12 +1444,12 @@ The argument is represented by a schema type, so can itself be a complex type.</
</span><span class=c1></span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>Schema</span> <span class=n>SCHEMA</span> <span class=o>=</span> <span class=n>Schema</span><span class=o>.</span><span class=na>builder</span><span class=o>().</span><span class=na>addInt64Field</span><span class=o>(</span><span class=s>"seconds"</span><span class=o>).</span><span class=na>addInt32Field</span><span class=o>(</span><span class=s>"nanos"</span><span class [...]
<span class=nd>@Override</span> <span class=kd>public</span> <span class=n>String</span> <span class=nf>getIdentifier</span><span class=o>()</span> <span class=o>{</span> <span class=k>return</span> <span class=s>"timestampNanos"</span><span class=o>;</span> <span class=o>}</span>
<span class=nd>@Override</span> <span class=kd>public</span> <span class=n>FieldType</span> <span class=nf>getBaseType</span><span class=o>()</span> <span class=o>{</span> <span class=k>return</span> <span class=n>schema</span><span class=o>;</span> <span class=o>}</span>
-
+
<span class=c1>// Convert the representation type to the underlying Row type. Called by Beam when necessary.
</span><span class=c1></span> <span class=nd>@Override</span> <span class=kd>public</span> <span class=n>Row</span> <span class=nf>toBaseType</span><span class=o>(</span><span class=n>Instant</span> <span class=n>instant</span><span class=o>)</span> <span class=o>{</span>
<span class=k>return</span> <span class=n>Row</span><span class=o>.</span><span class=na>withSchema</span><span class=o>(</span><span class=n>schema</span><span class=o>).</span><span class=na>addValues</span><span class=o>(</span><span class=n>instant</span><span class=o>.</span><span class=na>getEpochSecond</span><span class=o>(),</span> <span class=n>instant</span><span class=o>.</span><span class=na>getNano</span><span class=o>()).</span><span class=na>build</span><span class=o>( [...]
<span class=o>}</span>
-
+
<span class=c1>// Convert the underlying Row type to an Instant. Called by Beam when necessary.
</span><span class=c1></span> <span class=nd>@Override</span> <span class=kd>public</span> <span class=n>Instant</span> <span class=nf>toInputType</span><span class=o>(</span><span class=n>Row</span> <span class=n>base</span><span class=o>)</span> <span class=o>{</span>
<span class=k>return</span> <span class=n>Instant</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=n>row</span><span class=o>.</span><span class=na>getInt64</span><span class=o>(</span><span class=s>"seconds"</span><span class=o>),</span> <span class=n>row</span><span class=o>.</span><span class=na>getInt32</span><span class=o>(</span><span class=s>"nanos"</span><span class=o>));</span>
@@ -1467,14 +1465,14 @@ you access the enumeration either as a string or a value. For example:</p><div c
</span><span class=c1></span><span class=n>enumValue</span><span class=o>.</span><span class=na>toString</span><span class=o>();</span> <span class=o>//</span> <span class=n>Returns</span> <span class=s>"RED"</span><span class=o>,</span> <span class=n>the</span> <span class=n>string</span> <span class=n>value</span> <span class=n>of</span> <span class=n>the</span> <span class=n>constant</span></code></pre></div></div><p>Given a row object with an enumeration field, you can also [...]
types.</p><h5 id=oneoftype><strong>OneOfType</strong></h5><p>OneOfType allows creating a disjoint union type over a set of schema fields. For example:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=n>Schema</span> <span class=n>schema</span> <span class=o>=</span> <span class=n>Schema</span><span class=o>.</span><span class=na>builder</span><span class=o>()</span>
<span class=err>…</span>
- <span class=o>.</span><span class=na>addLogicalTypeField</span><span class=o>(</span><span class=s>"oneOfField"</span><span class=o>,</span>
+ <span class=o>.</span><span class=na>addLogicalTypeField</span><span class=o>(</span><span class=s>"oneOfField"</span><span class=o>,</span>
<span class=n>OneOfType</span><span class=o>.</span><span class=na>create</span><span class=o>(</span><span class=n>Field</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=s>"intField"</span><span class=o>,</span> <span class=n>FieldType</span><span class=o>.</span><span class=na>INT32</span><span class=o>),</span>
<span class=n>Field</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=s>"stringField"</span><span class=o>,</span> <span class=n>FieldType</span><span class=o>.</span><span class=na>STRING</span><span class=o>),</span>
<span class=n>Field</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=s>"bytesField"</span><span class=o>,</span> <span class=n>FieldType</span><span class=o>.</span><span class=na>BYTES</span><span class=o>)))</span>
<span class=o>.</span><span class=na>build</span><span class=o>();</span></code></pre></div></div><p>The value of this field is stored in the row as another Row type, where all the fields are marked as nullable. The
logical type however defines a Value object that contains an enumeration value indicating which field was set and allows
getting just that field:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=c1>// Returns an enumeration indicating all possible case values for the enum.
-</span><span class=c1>// For the above example, this will be
+</span><span class=c1>// For the above example, this will be
</span><span class=c1>// EnumerationType.create("intField", "stringField", "bytesField");
</span><span class=c1></span><span class=n>EnumerationType</span> <span class=n>oneOfEnum</span> <span class=o>=</span> <span class=n>onOfType</span><span class=o>.</span><span class=na>getCaseEnumType</span><span class=o>();</span>
@@ -1483,7 +1481,7 @@ getting just that field:</p><div class=language-java><div class=highlight><pre c
<span class=c1>// Handle the oneof
</span><span class=c1></span><span class=k>switch</span> <span class=o>(</span><span class=n>oneOfValue</span><span class=o>.</span><span class=na>getCaseEnumType</span><span class=o>().</span><span class=na>toString</span><span class=o>())</span> <span class=o>{</span>
- <span class=k>case</span> <span class=s>"intField"</span><span class=o>:</span>
+ <span class=k>case</span> <span class=s>"intField"</span><span class=o>:</span>
<span class=k>return</span> <span class=n>processInt</span><span class=o>(</span><span class=n>oneOfValue</span><span class=o>.</span><span class=na>getValue</span><span class=o>(</span><span class=n>Integer</span><span class=o>.</span><span class=na>class</span><span class=o>));</span>
<span class=k>case</span> <span class=s>"stringField"</span><span class=o>:</span>
<span class=k>return</span> <span class=n>processString</span><span class=o>(</span><span class=n>oneOfValue</span><span class=o>.</span><span class=na>getValue</span><span class=o>(</span><span class=n>String</span><span class=o>.</span><span class=na>class</span><span class=o>));</span>
@@ -1515,12 +1513,12 @@ schema. For example, it’s common to have ephemeral fields in a class that shou
(e.g. caching the hash value to prevent expensive recomputation of the hash), and <code>@SchemaIgnore</code> can be used to
exclude these fields. Note that ignored fields will not be included in the encoding of these records.</p><p>In some cases it is not convenient to annotate the POJO class, for example if the POJO is in a different package that is
not owned by the Beam pipeline author. In these cases the schema inference can be triggered programmatically in
-pipeline’s main function as follows:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java> <span class=n>pipeline</span><span class=o>.</span><span class=na>getSchemaRegistry</span><span class=o>().</span><span class=na>registerPOJO</span><span class=o>(</span><span class=n>TransactionPOJO</span><span class=o>.</span><span class=na>class</span><span class=o>);</span> </code></pre></div></div><h5 id=java-beans><strong>Java Beans</stro [...]
+pipeline’s main function as follows:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java> <span class=n>pipeline</span><span class=o>.</span><span class=na>getSchemaRegistry</span><span class=o>().</span><span class=na>registerPOJO</span><span class=o>(</span><span class=n>TransactionPOJO</span><span class=o>.</span><span class=na>class</span><span class=o>);</span></code></pre></div></div><h5 id=java-beans><strong>Java Beans</stron [...]
standard has many characteristics, the key ones are that all properties are accessed via getter and setter classes, and
the name format for these getters and setters is standardized. A Java Bean class can be annotated with
<code>@DefaultSchema(JavaBeanSchema.class)</code> and Beam will automatically infer a schema for this class. For example:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=nd>@DefaultSchema</span><span class=o>(</span><span class=n>JavaBeanSchema</span><span class=o>.</span><span class=na>class</span><span class=o>)</span>
<span class=kd>public</span> <span class=kd>class</span> <span class=nc>TransactionBean</span> <span class=o>{</span>
- <span class=kd>public</span> <span class=nf>TransactionBean</span><span class=o>()</span> <span class=o>{</span> <span class=err>…</span> <span class=o>}</span>
+ <span class=kd>public</span> <span class=nf>TransactionBean</span><span class=o>()</span> <span class=o>{</span> <span class=err>…</span> <span class=o>}</span>
<span class=kd>public</span> <span class=n>String</span> <span class=nf>getBank</span><span class=o>()</span> <span class=o>{</span> <span class=err>…</span> <span class=o>}</span>
<span class=kd>public</span> <span class=kt>void</span> <span class=nf>setBank</span><span class=o>(</span><span class=n>String</span> <span class=n>bank</span><span class=o>)</span> <span class=o>{</span> <span class=err>…</span> <span class=o>}</span>
<span class=kd>public</span> <span class=kt>double</span> <span class=nf>getPurchaseAmount</span><span class=o>()</span> <span class=o>{</span> <span class=err>…</span> <span class=o>}</span>
@@ -1539,7 +1537,7 @@ order to properly implement a value class. AutoValue is a popular library for ea
implementing a simple abstract base class.</p><p>Beam can infer a schema from an AutoValue class. For example:</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=nd>@DefaultSchema</span><span class=o>(</span><span class=n>AutoValueSchema</span><span class=o>.</span><span class=na>class</span><span class=o>)</span>
<span class=nd>@AutoValue</span>
<span class=kd>public</span> <span class=kd>abstract</span> <span class=kd>class</span> <span class=nc>TransactionValue</span> <span class=o>{</span>
- <span class=kd>public</span> <span class=kd>abstract</span> <span class=n>String</span> <span class=nf>getBank</span><span class=o>();</span>
+ <span class=kd>public</span> <span class=kd>abstract</span> <span class=n>String</span> <span class=nf>getBank</span><span class=o>();</span>
<span class=kd>public</span> <span class=kd>abstract</span> <span class=kt>double</span> <span class=nf>getPurchaseAmount</span><span class=o>();</span>
<span class=o>}</span></code></pre></div></div><p>This is all that’s needed to generate a simple AutoValue class, and the above <code>@DefaultSchema</code> annotation tells Beam to
infer a schema from it. This also allows AutoValue elements to be used inside of <code>PCollection</code>s.</p><p><code>@SchemaFieldName</code> and <code>@SchemaIgnore</code> can be used to alter the schema inferred.</p><h3 id=using-schemas>6.6. Using Schema Transforms</h3><p>A schema on a <code>PCollection</code> enables a rich variety of relational transforms. The fact that each record is composed of
@@ -1627,7 +1625,7 @@ renamed using the field-selection syntax.</p><p>For example, the following snipp
<span class=o>.</span><span class=na>rename</span><span class=o>(</span><span class=s>"shippingAddress.streetAddress"</span><span class=o>,</span> <span class=s>"shippingAddress.street"</span><span class=o>));</span></code></pre></div></div><p>Results in the same set of unmodified input elements, however the schema on the PCollection has been changed to rename
<strong>userId</strong> to <strong>userIdentifier</strong> and <strong>shippingAddress.streetAddress</strong> to <strong>shippingAddress.street</strong>.</p><h5 id=converting-between-types><strong>Converting between types</strong></h5><p>As mentioned, Beam can automatically convert between different Java types, as long as those types have equivalent
schemas. One way to do this is by using the <code>Convert</code> transform, as follows.</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=n>PCollection</span><span class=o><</span><span class=n>PurchaseBean</span><span class=o>></span> <span class=n>purchaseBeans</span> <span class=o>=</span> <span class=n>readPurchasesAsBeans</span><span class=o>();</span>
-<span class=n>PCollection</span><span class=o><</span><span class=n>PurchasePojo</span><span class=o>></span> <span class=n>pojoPurchases</span> <span class=o>=</span>
+<span class=n>PCollection</span><span class=o><</span><span class=n>PurchasePojo</span><span class=o>></span> <span class=n>pojoPurchases</span> <span class=o>=</span>
<span class=n>purchaseBeans</span><span class=o>.</span><span class=na>apply</span><span class=o>(</span><span class=n>Convert</span><span class=o>.</span><span class=na>to</span><span class=o>(</span><span class=n>PurchasePojo</span><span class=o>.</span><span class=na>class</span><span class=o>));</span></code></pre></div></div><p>Beam will validate that the inferred schema for <code>PurchasePojo</code> matches that of the input <code>PCollection</code>, and will
then cast to a <code>PCollection<PurchasePojo></code>.</p><p>Since the <code>Row</code> class can support any schema, any <code>PCollection</code> with schema can be cast to a <code>PCollection</code> of rows, as
follows.</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=n>PCollection</span><span class=o><</span><span class=n>Row</span><span class=o>></span> <span class=n>purchaseRows</span> <span class=o>=</span> <span class=n>purchaseBeans</span><span class=o>.</span><span class=na>apply</span><span class=o>(</span><span class=n>Convert</span><span class=o>.</span><span class=na>toRows</span><span class=o>());</span></c [...]
@@ -2089,8 +2087,7 @@ windowing function:</p><div class=language-java><div class=highlight><pre class=
<span class=n>FixedWindows</span><span class=p>(</span><span class=mi>60</span><span class=p>),</span>
<span class=n>trigger</span><span class=o>=</span><span class=n>AfterProcessingTime</span><span class=p>(</span><span class=mi>60</span><span class=p>),</span>
<span class=n>allowed_lateness</span><span class=o>=</span><span class=mi>1800</span><span class=p>)</span> <span class=c1># 30 minutes</span>
- <span class=o>|</span> <span class=o>...</span>
- </code></pre></div></div><p>This allowed lateness propagates to all <code>PCollection</code>s derived as a result of
+ <span class=o>|</span> <span class=o>...</span></code></pre></div></div><p>This allowed lateness propagates to all <code>PCollection</code>s derived as a result of
applying transforms to the original <code>PCollection</code>. If you want to change the
allowed lateness later in your pipeline, you can apply
<code>Window.configure().withAllowedLateness()</code> again, explicitly.</p><h3 id=composite-triggers>9.5. Composite triggers</h3><p>You can combine multiple triggers to form <strong>composite triggers</strong>, and can
@@ -2158,7 +2155,7 @@ after jobs have completed.</p></blockquote><h3 id=types-of-metrics>10.2 Types of
<span class=nd>@ProcessElement</span>
<span class=kd>public</span> <span class=kt>void</span> <span class=nf>processElement</span><span class=o>(</span><span class=n>ProcessContext</span> <span class=n>context</span><span class=o>)</span> <span class=o>{</span>
<span class=n>Integer</span> <span class=n>element</span> <span class=o>=</span> <span class=n>context</span><span class=o>.</span><span class=na>element</span><span class=o>();</span>
- <span class=c1>// create a distribution (histogram) of the values
+ <span class=c1>// create a distribution (histogram) of the values
</span><span class=c1></span> <span class=n>distribution</span><span class=o>.</span><span class=na>update</span><span class=o>(</span><span class=n>element</span><span class=o>);</span>
<span class=o>...</span>
<span class=o>}</span></code></pre></div></div><p><strong>Gauge</strong>: A metric that reports the latest value out of reported values. Since metrics are
@@ -2167,7 +2164,7 @@ collected from many workers the value may not be the absolute last, but one of t
<span class=nd>@ProcessElement</span>
<span class=kd>public</span> <span class=kt>void</span> <span class=nf>processElement</span><span class=o>(</span><span class=n>ProcessContext</span> <span class=n>context</span><span class=o>)</span> <span class=o>{</span>
<span class=n>Integer</span> <span class=n>element</span> <span class=o>=</span> <span class=n>context</span><span class=o>.</span><span class=na>element</span><span class=o>();</span>
- <span class=c1>// create a gauge (latest value received) of the values
+ <span class=c1>// create a gauge (latest value received) of the values
</span><span class=c1></span> <span class=n>gauge</span><span class=o>.</span><span class=na>set</span><span class=o>(</span><span class=n>element</span><span class=o>);</span>
<span class=o>...</span>
<span class=o>}</span></code></pre></div></div><h3 id=querying-metrics>10.3 Querying metrics</h3><p><code>PipelineResult</code> has a method <code>metrics()</code> which returns a <code>MetricResults</code> object that allows
@@ -2207,7 +2204,7 @@ matching a given filter.</p><div class=language-java><div class=highlight><pre c
<span class=o>.</span><span class=na>addNameFilter</span><span class=o>(</span><span class=n>MetricNameFilter</span><span class=o>.</span><span class=na>named</span><span class=o>(</span><span class=s>"namespace"</span><span class=o>,</span> <span class=s>"counter1"</span><span class=o>))</span>
<span class=o>.</span><span class=na>build</span><span class=o>());</span>
-<span class=c1>// print the metric value - there should be only one line because there is only one metric
+<span class=c1>// print the metric value - there should be only one line because there is only one metric
</span><span class=c1>// called "counter1" in the namespace called "namespace"
</span><span class=c1></span><span class=k>for</span> <span class=o>(</span><span class=n>MetricResult</span><span class=o><</span><span class=n>Long</span><span class=o>></span> <span class=n>counter</span><span class=o>:</span> <span class=n>metrics</span><span class=o>.</span><span class=na>getCounters</span><span class=o>())</span> <span class=o>{</span>
<span class=n>System</span><span class=o>.</span><span class=na>out</span><span class=o>.</span><span class=na>println</span><span class=o>(</span><span class=n>counter</span><span class=o>.</span><span class=na>getName</span><span class=o>()</span> <span class=o>+</span> <span class=s>":"</span> <span class=o>+</span> <span class=n>counter</span><span class=o>.</span><span class=na>getAttempted</span><span class=o>());</span>
@@ -2249,7 +2246,7 @@ specified when creating the ValueState. For example, the following ParDo creates
accumulates the number of elements seen.</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=n>PCollection</span><span class=o><</span><span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>>></span> <span class=n>perUser</span> <span class=o>=</span> <span class=n>readPerUser</span><span class=o>();</span>
<span class=n>perUser</span><span class=o>.</span><span class=na>apply</span><span class=o>(</span><span class=n>ParDo</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=k>new</span> <span class=n>DoFn</span><span class=o><</span><span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>>,</span> <span class=n>OutputT</span><span class=o>>()</span> <span class=o>{</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>ValueState</span><span class=o><</span><span class=n>Integer</span><span class=o>>></span> <span class=n>numElements</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>value</span><span class=o>() [...]
-
+
<span class=nd>@ProcessElement</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>process</span><span class=o>(</span><span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Integer</span><span class=o>></span> <span class=n>state</span><span class=o>)</span> <span class=o>{</span>
<span class=c1>// Read the number element seen so far for this user key.
</span><span class=c1></span> <span class=c1>// state.read() returns null if it was never set. The below code allows us to have a default value of 0.
@@ -2264,18 +2261,18 @@ accumulates the number of elements seen.</p><div class=language-java><div class=
<span class=o>}));</span></code></pre></div></div><h4 id=combiningstate>CombiningState</h4><p><code>CombiningState</code> allows you to create a state object that is updated using a Beam combiner. For example, the previous
<code>ValueState</code> example could be rewritten to use <code>CombiningState</code></p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=n>PCollection</span><span class=o><</span><span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>>></span> <span class=n>perUser</span> <span class=o>=</span> <span class=n>readPerUser</span><s [...]
<span class=n>perUser</span><span class=o>.</span><span class=na>apply</span><span class=o>(</span><span class=n>ParDo</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=k>new</span> <span class=n>DoFn</span><span class=o><</span><span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>>,</span> <span class=n>OutputT</span><span class=o>>()</span> <span class=o>{</span>
- <span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>CombiningState</span><span class=o><</span><span class=n>Integer</span><span class=o>,</span> <span class=kt>int</span><span class=o>[],</span> <span class=n>Integer</span><span class=o>>></span> <span class=n>numElements</span> <span class=o> [...]
+ <span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>CombiningState</span><span class=o><</span><span class=n>Integer</span><span class=o>,</span> <span class=kt>int</span><span class=o>[],</span> <span class=n>Integer</span><span class=o>>></span> <span class=n>numElements</span> <span class=o> [...]
<span class=n>StateSpecs</span><span class=o>.</span><span class=na>combining</span><span class=o>(</span><span class=n>Sum</span><span class=o>.</span><span class=na>ofIntegers</span><span class=o>());</span>
-
+
<span class=nd>@ProcessElement</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>process</span><span class=o>(</span><span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Integer</span><span class=o>></span> <span class=n>state</span><span class=o>)</span> <span class=o>{</span>
<span class=n>state</span><span class=o>.</span><span class=na>add</span><span class=o>(</span><span class=n>1</span><span class=o>);</span>
<span class=o>}</span>
<span class=o>}));</span></code></pre></div></div><div class=language-python><div class=highlight><pre class=chroma><code class=language-python data-lang=python><span class=k>class</span> <span class=nc>CombiningStateDoFn</span><span class=p>(</span><span class=n>DoFn</span><span class=p>):</span>
<span class=n>SUM_TOTAL</span> <span class=o>=</span> <span class=n>CombiningValueStateSpec</span><span class=p>(</span><span class=s1>'total'</span><span class=p>,</span> <span class=nb>sum</span><span class=p>)</span>
-
+
<span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=n>element</span><span class=p>,</span> <span class=n>state</span><span class=o>=</span><span class=n>SoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>SUM_TOTAL</span><span class=p>)):</span>
<span class=n>state</span><span class=o>.</span><span class=n>add</span><span class=p>(</span><span class=mi>1</span><span class=p>)</span>
-
+
<span class=n>_</span> <span class=o>=</span> <span class=p>(</span><span class=n>p</span> <span class=o>|</span> <span class=s1>'Read per user'</span> <span class=o>>></span> <span class=n>ReadPerUser</span><span class=p>()</span>
<span class=o>|</span> <span class=s1>'Combine state pardo'</span> <span class=o>>></span> <span class=n>beam</span><span class=o>.</span><span class=n>ParDo</span><span class=p>(</span><span class=n>CombiningStateDofn</span><span class=p>()))</span></code></pre></div></div><h4 id=bagstate>BagState</h4><p>A common use case for state is to accumulate multiple elements. <code>BagState</code> allows for accumulating an unordered set
of elements. This allows for addition of elements to the collection without requiring the reading of the entire
@@ -2283,9 +2280,9 @@ collection first, which is an efficiency gain. In addition, runners that support
bags larger than available memory.</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=n>PCollection</span><span class=o><</span><span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>>></span> <span class=n>perUser</span> <span class=o>=</span> <span class=n>readPerUser</span><span class=o>();</span>
<span class=n>perUser</span><span class=o>.</span><span class=na>apply</span><span class=o>(</span><span class=n>ParDo</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=k>new</span> <span class=n>DoFn</span><span class=o><</span><span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>>,</span> <span class=n>OutputT</span><span class=o>>()</span> <span class=o>{</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>BagState</span><span class=o><</span><span class=n>ValueT</span><span class=o>>></span> <span class=n>numElements</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>bag</span><span class=o>();</span>
-
+
<span class=nd>@ProcessElement</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>process</span><span class=o>(</span>
- <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
+ <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=n>BagState</span><span class=o><</span><span class=n>ValueT</span><span class=o>></span> <span class=n>state</span><span class=o>)</span> <span class=o>{</span>
<span class=c1>// Add the current element to the bag for this key.
</span><span class=c1></span> <span class=n>state</span><span class=o>.</span><span class=na>add</span><span class=o>(</span><span class=n>element</span><span class=o>.</span><span class=na>getValue</span><span class=o>());</span>
@@ -2298,14 +2295,14 @@ bags larger than available memory.</p><div class=language-java><div class=highli
<span class=o>}</span>
<span class=o>}));</span></code></pre></div></div><div class=language-python><div class=highlight><pre class=chroma><code class=language-python data-lang=python><span class=k>class</span> <span class=nc>BagStateDoFn</span><span class=p>(</span><span class=n>DoFn</span><span class=p>):</span>
<span class=n>ALL_ELEMENTS</span> <span class=o>=</span> <span class=n>BagStateSpec</span><span class=p>(</span><span class=s1>'buffer'</span><span class=p>,</span> <span class=n>coders</span><span class=o>.</span><span class=n>VarIntCoder</span><span class=p>())</span>
-
+
<span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=n>element_pair</span><span class=p>,</span> <span class=n>state</span><span class=o>=</span><span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>)):</span>
<span class=n>state</span><span class=o>.</span><span class=n>add</span><span class=p>(</span><span class=n>element_pair</span><span class=p>[</span><span class=mi>1</span><span class=p>])</span>
<span class=k>if</span> <span class=n>should_fetch</span><span class=p>():</span>
<span class=n>all_elements</span> <span class=o>=</span> <span class=nb>list</span><span class=p>(</span><span class=n>state</span><span class=o>.</span><span class=n>read</span><span class=p>())</span>
<span class=n>process_values</span><span class=p>(</span><span class=n>all_elements</span><span class=p>)</span>
<span class=n>state</span><span class=o>.</span><span class=n>clear</span><span class=p>()</span>
-
+
<span class=n>_</span> <span class=o>=</span> <span class=p>(</span><span class=n>p</span> <span class=o>|</span> <span class=s1>'Read per user'</span> <span class=o>>></span> <span class=n>ReadPerUser</span><span class=p>()</span>
<span class=o>|</span> <span class=s1>'Bag state pardo'</span> <span class=o>>></span> <span class=n>beam</span><span class=o>.</span><span class=n>ParDo</span><span class=p>(</span><span class=n>BagStateDoFn</span><span class=p>()))</span></code></pre></div></div><h3 id=deferred-state-reads>11.2 Deferred state reads</h3><p>When a <code>DoFn</code> contains multiple state specifications, reading each one in order can be slow. Calling the <code>read()</code> function
on a state can cause the runner to perform a blocking read. Performing multiple blocking reads in sequence adds latency
@@ -2341,7 +2338,7 @@ be read in the future, allowing multiple state reads to be batched together.</p>
<span class=n>state2</span><span class=o>.</span><span class=na>readLater</span><span class=o>();</span>
<span class=n>state3</span><span class=o>.</span><span class=na>readLater</span><span class=o>();</span>
<span class=o>}</span>
-
+
<span class=c1>// The runner can now batch all three states into a single read, reducing latency.
</span><span class=c1></span> <span class=n>processState1</span><span class=o>(</span><span class=n>state1</span><span class=o>.</span><span class=na>read</span><span class=o>());</span>
<span class=n>processState2</span><span class=o>(</span><span class=n>state2</span><span class=o>.</span><span class=na>read</span><span class=o>());</span>
@@ -2360,33 +2357,33 @@ allows for event-time aggregations.</p><div class=language-java><div class=highl
<span class=nd>@ProcessElement</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>process</span><span class=o>(</span>
<span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
<span class=nd>@Timestamp</span> <span class=n>Instant</span> <span class=n>elementTs</span><span class=o>,</span>
- <span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Integer</span><span class=o>></span> <span class=n>state</span><span class=o>,</span>
+ <span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Integer</span><span class=o>></span> <span class=n>state</span><span class=o>,</span>
<span class=nd>@TimerId</span><span class=o>(</span><span class=s>"timer"</span><span class=o>)</span> <span class=n>Timer</span> <span class=n>timer</span><span class=o>)</span> <span class=o>{</span>
<span class=o>...</span>
<span class=c1>// Set an event-time timer to the element timestamp.
</span><span class=c1></span> <span class=n>timer</span><span class=o>.</span><span class=na>set</span><span class=o>(</span><span class=n>elementTs</span><span class=o>);</span>
<span class=o>}</span>
-
+
<span class=nd>@OnTimer</span><span class=o>(</span><span class=s>"timer"</span><span class=o>)</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>onTimer</span><span class=o>()</span> <span class=o>{</span>
<span class=c1>//Process timer.
</span><span class=c1></span> <span class=o>}</span>
<span class=o>}));</span></code></pre></div></div><div class=language-python><div class=highlight><pre class=chroma><code class=language-python data-lang=python><span class=k>class</span> <span class=nc>EventTimerDoFn</span><span class=p>(</span><span class=n>DoFn</span><span class=p>):</span>
<span class=n>ALL_ELEMENTS</span> <span class=o>=</span> <span class=n>BagStateSpec</span><span class=p>(</span><span class=s1>'buffer'</span><span class=p>,</span> <span class=n>coders</span><span class=o>.</span><span class=n>VarIntCoder</span><span class=p>())</span>
<span class=n>TIMER</span> <span class=o>=</span> <span class=n>TimerSpec</span><span class=p>(</span><span class=s1>'timer'</span><span class=p>,</span> <span class=n>TimeDomain</span><span class=o>.</span><span class=n>WATERMARK</span><span class=p>)</span>
-
- <span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
- <span class=n>element_pair</span><span class=p>,</span>
+
+ <span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
+ <span class=n>element_pair</span><span class=p>,</span>
<span class=n>t</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>TimestampParam</span><span class=p>,</span>
- <span class=nb>buffer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>),</span>
+ <span class=nb>buffer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>),</span>
<span class=n>timer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>TimerParam</span><span class=p>(</span><span class=n>TIMER</span><span class=p>)):</span>
<span class=nb>buffer</span><span class=o>.</span><span class=n>add</span><span class=p>(</span><span class=n>element_pair</span><span class=p>[</span><span class=mi>1</span><span class=p>])</span>
<span class=c1># Set an event-time timer to the element timestamp.</span>
<span class=n>timer</span><span class=o>.</span><span class=n>set</span><span class=p>(</span><span class=n>t</span><span class=p>)</span>
-
+
<span class=nd>@on_timer</span><span class=p>(</span><span class=n>TIMER</span><span class=p>)</span>
<span class=k>def</span> <span class=nf>expiry_callback</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=nb>buffer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>)):</span>
<span class=n>state</span><span class=o>.</span><span class=n>clear</span><span class=p>()</span>
-
+
<span class=n>_</span> <span class=o>=</span> <span class=p>(</span><span class=n>p</span> <span class=o>|</span> <span class=s1>'Read per user'</span> <span class=o>>></span> <span class=n>ReadPerUser</span><span class=p>()</span>
<span class=o>|</span> <span class=s1>'EventTime timer pardo'</span> <span class=o>>></span> <span class=n>beam</span><span class=o>.</span><span class=n>ParDo</span><span class=p>(</span><span class=n>EventTimerDoFn</span><span class=p>()))</span></code></pre></div></div><h4 id=processing-time-timers>11.3.2 Processing-time timers</h4><p>Processing-time timers fire when the real wall-clock time passes. This is often used to create larger batches of data
before processing. It can also be used to schedule events that should occur at a specific time. Just like with
@@ -2400,27 +2397,27 @@ to the current time. In Java, the <code>Timer.offset</code> and <code>Timer.setR
<span class=c1>// Set a timer to go off 30 seconds in the future.
</span><span class=c1></span> <span class=n>timer</span><span class=o>.</span><span class=na>offset</span><span class=o>(</span><span class=n>Duration</span><span class=o>.</span><span class=na>standardSeconds</span><span class=o>(</span><span class=n>30</span><span class=o>)).</span><span class=na>setRelative</span><span class=o>();</span>
<span class=o>}</span>
-
+
<span class=nd>@OnTimer</span><span class=o>(</span><span class=s>"timer"</span><span class=o>)</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>onTimer</span><span class=o>()</span> <span class=o>{</span>
<span class=c1>//Process timer.
</span><span class=c1></span> <span class=o>}</span>
<span class=o>}));</span></code></pre></div></div><div class=language-python><div class=highlight><pre class=chroma><code class=language-python data-lang=python><span class=k>class</span> <span class=nc>ProcessingTimerDoFn</span><span class=p>(</span><span class=n>DoFn</span><span class=p>):</span>
<span class=n>ALL_ELEMENTS</span> <span class=o>=</span> <span class=n>BagStateSpec</span><span class=p>(</span><span class=s1>'buffer'</span><span class=p>,</span> <span class=n>coders</span><span class=o>.</span><span class=n>VarIntCoder</span><span class=p>())</span>
<span class=n>TIMER</span> <span class=o>=</span> <span class=n>TimerSpec</span><span class=p>(</span><span class=s1>'timer'</span><span class=p>,</span> <span class=n>TimeDomain</span><span class=o>.</span><span class=n>REAL_TIME</span><span class=p>)</span>
-
- <span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
- <span class=n>element_pair</span><span class=p>,</span>
- <span class=nb>buffer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>),</span>
+
+ <span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
+ <span class=n>element_pair</span><span class=p>,</span>
+ <span class=nb>buffer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>),</span>
<span class=n>timer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>TimerParam</span><span class=p>(</span><span class=n>TIMER</span><span class=p>)):</span>
<span class=nb>buffer</span><span class=o>.</span><span class=n>add</span><span class=p>(</span><span class=n>element_pair</span><span class=p>[</span><span class=mi>1</span><span class=p>])</span>
<span class=c1># Set a timer to go off 30 seconds in the future.</span>
<span class=n>timer</span><span class=o>.</span><span class=n>set</span><span class=p>(</span><span class=n>Timestamp</span><span class=o>.</span><span class=n>now</span><span class=p>()</span> <span class=o>+</span> <span class=n>Duration</span><span class=p>(</span><span class=n>seconds</span><span class=o>=</span><span class=mi>30</span><span class=p>))</span>
-
+
<span class=nd>@on_timer</span><span class=p>(</span><span class=n>TIMER</span><span class=p>)</span>
<span class=k>def</span> <span class=nf>expiry_callback</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=nb>buffer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>)):</span>
<span class=c1># Process timer.</span>
<span class=n>state</span><span class=o>.</span><span class=n>clear</span><span class=p>()</span>
-
+
<span class=n>_</span> <span class=o>=</span> <span class=p>(</span><span class=n>p</span> <span class=o>|</span> <span class=s1>'Read per user'</span> <span class=o>>></span> <span class=n>ReadPerUser</span><span class=p>()</span>
<span class=o>|</span> <span class=s1>'ProcessingTime timer pardo'</span> <span class=o>>></span> <span class=n>beam</span><span class=o>.</span><span class=n>ParDo</span><span class=p>(</span><span class=n>ProcessingTimerDoFn</span><span class=p>()))</span></code></pre></div></div><h4 id=dynamic-timer-tags>11.3.3 Dynamic timer tags</h4><p>Beam also supports dynamically setting a timer tag using <code>TimerMap</code>. This allows for setting multiple different timers
in a <code>DoFn</code> and allowing for the timer tags to be dynamically chosen - e.g. based on data in the input elements. A
@@ -2432,17 +2429,18 @@ id, and timers in different timer families are independent.</p><div class=langua
<span class=n>TimerSpecs</span><span class=o>.</span><span class=na>timerMap</span><span class=o>(</span><span class=n>TimeDomain</span><span class=o>.</span><span class=na>EVENT_TIME</span><span class=o>);</span>
<span class=nd>@ProcessElement</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>process</span><span class=o>(</span>
- <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
+ <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
<span class=nd>@Timestamp</span> <span class=n>Instant</span> <span class=n>elementTs</span><span class=o>,</span>
<span class=nd>@TimerFamily</span><span class=o>(</span><span class=s>"actionTimers"</span><span class=o>)</span> <span class=n>TimerMap</span> <span class=n>timers</span><span class=o>)</span> <span class=o>{</span>
<span class=n>timers</span><span class=o>.</span><span class=na>set</span><span class=o>(</span><span class=n>element</span><span class=o>.</span><span class=na>getValue</span><span class=o>().</span><span class=na>getActionType</span><span class=o>(),</span> <span class=n>elementTs</span><span class=o>);</span>
<span class=o>}</span>
-
+
<span class=nd>@OnTimerFamily</span><span class=o>(</span><span class=s>"actionTimers"</span><span class=o>)</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>onTimer</span><span class=o>(</span><span class=nd>@TimerId</span> <span class=n>String</span> <span class=n>timerId</span><span class=o>)</span> <span class=o>{</span>
<span class=n>LOG</span><span class=o>.</span><span class=na>info</span><span class=o>(</span><span class=s>"Timer fired with id "</span> <span class=o>+</span> <span class=n>timerId</span><span class=o>);</span>
<span class=o>}</span>
<span class=o>}));</span></code></pre></div></div><div class=language-python><div class=highlight><pre class=chroma><code class=language-python data-lang=python><span class=n>To</span> <span class=n>be</span> <span class=n>supported</span><span class=p>,</span> <span class=n>See</span> <span class=n>BEAM</span><span class=o>-</span><span class=mi>9602</span></code></pre></div></div><h4 id=timer-output-timestamps>11.3.4 Timer output timestamps</h4><p>By default, event-time timers will hol [...]
-that if a timer is set to 12pm, any windowed aggregations or event-time timers later in the pipeline graph that finish<br>after 12pm will not expire. The timestamp of the timer is also the default output timestamp for the timer callback. This
+that if a timer is set to 12pm, any windowed aggregations or event-time timers later in the pipeline graph that finish
+after 12pm will not expire. The timestamp of the timer is also the default output timestamp for the timer callback. This
means that any elements output from the onTimer method will have a timestamp equal to the timestamp of the timer firing.
For processing-time timers, the default output timestamp and watermark hold is the value of the input watermark at the
time the timer was set.</p><p>In some cases, a DoFn needs to output timestamps earlier than the timer expiration time, and therefore also needs to
@@ -2452,9 +2450,9 @@ records into state, and sets a timer to drain the state. This code may appear co
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"elementBag"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>BagState</span><span class=o><</span><span class=n>ValueT</span><span class=o>>></span> <span class=n>elementBag</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>bag</span><span class=o>();</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"timerSet"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>ValueState</span><span class=o><</span><span class=n>Boolean</span><span class=o>>></span> <span class=n>timerSet</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>value</span><span class=o>() [...]
<span class=nd>@TimerId</span><span class=o>(</span><span class=s>"outputState"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>TimerSpec</span> <span class=n>timer</span> <span class=o>=</span> <span class=n>TimerSpecs</span><span class=o>.</span><span class=na>timer</span><span class=o>(</span><span class=n>TimeDomain</span><span class=o>.</span><span class=na>PROCESSING_TIME</span><span class=o>);</span>
-
+
<span class=nd>@ProcessElement</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>process</span><span class=o>(</span>
- <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
+ <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"elementBag"</span><span class=o>)</span> <span class=n>BagState</span><span class=o><</span><span class=n>ValueT</span><span class=o>></span> <span class=n>elementBag</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"timerSet"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Boolean</span><span class=o>></span> <span class=n>timerSet</span><span class=o>,</span>
<span class=nd>@TimerId</span><span class=o>(</span><span class=s>"outputState"</span><span class=o>)</span> <span class=n>Timer</span> <span class=n>timer</span><span class=o>)</span> <span class=o>{</span>
@@ -2466,7 +2464,7 @@ records into state, and sets a timer to drain the state. This code may appear co
<span class=n>timerSet</span><span class=o>.</span><span class=na>write</span><span class=o>(</span><span class=kc>true</span><span class=o>);</span>
<span class=o>}</span>
<span class=o>}</span>
-
+
<span class=nd>@OnTimer</span><span class=o>(</span><span class=s>"outputState"</span><span class=o>)</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>onTimer</span><span class=o>(</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"elementBag"</span><span class=o>)</span> <span class=n>BagState</span><span class=o><</span><span class=n>ValueT</span><span class=o>></span> <span class=n>elementBag</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"timerSet"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Boolean</span><span class=o>></span> <span class=n>timerSet</span><span class=o>,</span>
@@ -2489,13 +2487,13 @@ past the timestamp of the minimum element. The following code demonstrates this.
<span class=c1>// The timestamp of the timer set.
</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"timerTimestamp"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>ValueState</span><span class=o><</span><span class=n>Long</span><span class=o>>></span> <span class=n>timerTimestamp</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span [...]
<span class=c1>// The minimum timestamp stored in the bag.
-</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"minTimestampInBag"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>CombiningState</span><span class=o><</span><span class=n>Long</span><span class=o>,</span> <span class=kt>long</span><span class=o>[],</span> <span class=n>Long</span><span class=o>>></span>
+</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"minTimestampInBag"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>CombiningState</span><span class=o><</span><span class=n>Long</span><span class=o>,</span> <span class=kt>long</span><span class=o>[],</span> <span class=n>Long</span><span class=o>>></span>
<span class=n>minTimestampInBag</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>combining</span><span class=o>(</span><span class=n>Min</span><span class=o>.</span><span class=na>ofLongs</span><span class=o>());</span>
<span class=nd>@TimerId</span><span class=o>(</span><span class=s>"outputState"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>TimerSpec</span> <span class=n>timer</span> <span class=o>=</span> <span class=n>TimerSpecs</span><span class=o>.</span><span class=na>timer</span><span class=o>(</span><span class=n>TimeDomain</span><span class=o>.</span><span class=na>PROCESSING_TIME</span><span class=o>);</span>
-
+
<span class=nd>@ProcessElement</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>process</span><span class=o>(</span>
- <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
+ <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"elementBag"</span><span class=o>)</span> <span class=n>BagState</span><span class=o><</span><span class=n>ValueT</span><span class=o>></span> <span class=n>elementBag</span><span class=o>,</span>
<span class=nd>@AlwaysFetched</span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"timerTimestamp"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Long</span><span class=o>></span> <span class=n>timerTimestamp</span><span class=o>,</span>
<span class=nd>@AlwaysFetched</span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"minTimestampInBag"</span><span class=o>)</span> <span class=n>CombiningState</span><span class=o><</span><span class=n>Long</span><span class=o>,</span> <span class=kt>long</span><span class=o>[],</span> <span class=n>Long</span><span class=o>></span> <span class=n>minTimestamp</span><span class=o>,</span>
@@ -2515,7 +2513,7 @@ past the timestamp of the minimum element. The following code demonstrates this.
</span><span class=c1></span> <span class=n>timer</span><span class=o>.</span><span class=na>withOutputTimestamp</span><span class=o>(</span><span class=n>minTimestamp</span><span class=o>.</span><span class=na>read</span><span class=o>()).</span><span class=na>set</span><span class=o>(</span><span class=n>timerToSet</span><span class=o>).</span>
<span class=n>timerTimestamp</span><span class=o>.</span><span class=na>write</span><span class=o>(</span><span class=n>timerToSet</span><span class=o>.</span><span class=na>getMillis</span><span class=o>());</span>
<span class=o>}</span>
-
+
<span class=nd>@OnTimer</span><span class=o>(</span><span class=s>"outputState"</span><span class=o>)</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>onTimer</span><span class=o>(</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"elementBag"</span><span class=o>)</span> <span class=n>BagState</span><span class=o><</span><span class=n>ValueT</span><span class=o>></span> <span class=n>elementBag</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"timerTimestamp"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Long</span><span class=o>></span> <span class=n>timerTimestamp</span><span class=o>,</span>
@@ -2545,12 +2543,12 @@ garbage-collection strategy.</p><p>For example, given the following:</p><div cla
</span><span class=c1></span> <span class=o>}</span>
<span class=o>}));</span></code></pre></div></div><div class=language-python><div class=highlight><pre class=chroma><code class=language-python data-lang=python><span class=k>class</span> <span class=nc>StateDoFn</span><span class=p>(</span><span class=n>DoFn</span><span class=p>):</span>
<span class=n>ALL_ELEMENTS</span> <span class=o>=</span> <span class=n>BagStateSpec</span><span class=p>(</span><span class=s1>'buffer'</span><span class=p>,</span> <span class=n>coders</span><span class=o>.</span><span class=n>VarIntCoder</span><span class=p>())</span>
-
- <span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
- <span class=n>element_pair</span><span class=p>,</span>
+
+ <span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
+ <span class=n>element_pair</span><span class=p>,</span>
<span class=nb>buffer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>)):</span>
<span class=o>...</span>
-
+
<span class=n>_</span> <span class=o>=</span> <span class=p>(</span><span class=n>p</span> <span class=o>|</span> <span class=s1>'Read per user'</span> <span class=o>>></span> <span class=n>ReadPerUser</span><span class=p>()</span>
<span class=o>|</span> <span class=s1>'Windowing'</span> <span class=o>>></span> <span class=n>beam</span><span class=o>.</span><span class=n>WindowInto</span><span class=p>(</span><span class=n>FixedWindows</span><span class=p>(</span><span class=mi>60</span> <span class=o>*</span> <span class=mi>60</span> <span class=o>*</span> <span class=mi>24</span><span class=p>))</span>
<span class=o>|</span> <span class=s1>'DoFn'</span> <span class=o>>></span> <span class=n>beam</span><span class=o>.</span><span class=n>ParDo</span><span class=p>(</span><span class=n>StateDoFn</span><span class=p>()))</span></code></pre></div></div><p>This <code>ParDo</code> stores state per day. Once the pipeline is done processing data for a given day, all the state for that
@@ -2562,7 +2560,7 @@ This can be done by updating a timer that garbage collects state. For example</p
</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>ValueState</span><span class=o><</span><span class=n>ValueT</span><span class=o>>></span> <span class=n>state</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>value< [...]
<span class=c1>// The maximum element timestamp seen so far.
-</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"maxTimestampSeen"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>CombiningState</span><span class=o><</span><span class=n>Long</span><span class=o>,</span> <span class=kt>long</span><span class=o>[],</span> <span class=n>Long</span><span class=o>>></span>
+</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"maxTimestampSeen"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>CombiningState</span><span class=o><</span><span class=n>Long</span><span class=o>,</span> <span class=kt>long</span><span class=o>[],</span> <span class=n>Long</span><span class=o>>></span>
<span class=n>maxTimestamp</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>combining</span><span class=o>(</span><span class=n>Max</span><span class=o>.</span><span class=na>ofLongs</span><span class=o>());</span>
<span class=nd>@TimerId</span><span class=o>(</span><span class=s>"gcTimer"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>TimerSpec</span> <span class=n>gcTimer</span> <span class=o>=</span> <span class=n>TimerSpecs</span><span class=o>.</span><span class=na>timer</span><span class=o>(</span><span class=n>TimeDomain</span><span class=o>.</span><span class=na>EVENT_TIME</span><span class=o>);</span>
@@ -2572,11 +2570,11 @@ This can be done by updating a timer that garbage collects state. For example</p
<span class=nd>@Timestamp</span> <span class=n>Instant</span> <span class=n>ts</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>ValueT</span><span class=o>></span> <span class=n>state</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"maxTimestampSeen"</span><span class=o>)</span> <span class=n>CombiningState</span><span class=o><</span><span class=n>Long</span><span class=o>,</span> <span class=kt>long</span><span class=o>[],</span> <span class=n>Long</span><span class=o>></span> <span class=n>maxTimestamp</span><span class=o>,</span>
- <span class=nd>@TimerId</span><span class=o>(</span><span class=s>"gcTimer"</span><span class=o>)</span> <span class=n>gcTimer</span><span class=o>)</span> <span class=o>{</span>
+ <span class=nd>@TimerId</span><span class=o>(</span><span class=s>"gcTimer"</span><span class=o>)</span> <span class=n>gcTimer</span><span class=o>)</span> <span class=o>{</span>
<span class=n>updateState</span><span class=o>(</span><span class=n>state</span><span class=o>,</span> <span class=n>element</span><span class=o>);</span>
<span class=n>maxTimestamp</span><span class=o>.</span><span class=na>add</span><span class=o>(</span><span class=n>ts</span><span class=o>.</span><span class=na>getMillis</span><span class=o>());</span>
-
- <span class=c1>// Set the timer to be one hour after the maximum timestamp seen. This will keep overwriting the same timer, so
+
+ <span class=c1>// Set the timer to be one hour after the maximum timestamp seen. This will keep overwriting the same timer, so
</span><span class=c1></span> <span class=c1>// as long as there is activity on this key the state will stay active. Once the key goes inactive for one hour's
</span><span class=c1></span> <span class=c1>// worth of event time (as measured by the watermark), then the gc timer will fire.
</span><span class=c1></span> <span class=n>Instant</span> <span class=n>expirationTime</span> <span class=o>=</span> <span class=k>new</span> <span class=n>Instant</span><span class=o>(</span><span class=n>maxTimestamp</span><span class=o>.</span><span class=na>read</span><span class=o>()).</span><span class=na>plus</span><span class=o>(</span><span class=n>Duration</span><span class=o>.</span><span class=na>standardHours</span><span class=o>(</span><span class=n>1</span><span class= [...]
@@ -2594,30 +2592,30 @@ This can be done by updating a timer that garbage collects state. For example</p
<span class=n>ALL_ELEMENTS</span> <span class=o>=</span> <span class=n>BagStateSpec</span><span class=p>(</span><span class=s1>'state'</span><span class=p>,</span> <span class=n>coders</span><span class=o>.</span><span class=n>VarIntCoder</span><span class=p>())</span>
<span class=n>MAX_TIMESTAMP</span> <span class=o>=</span> <span class=n>CombiningValueStateSpec</span><span class=p>(</span><span class=s1>'max_timestamp_seen'</span><span class=p>,</span> <span class=nb>max</span><span class=p>)</span>
<span class=n>TIMER</span> <span class=o>=</span> <span class=n>TimerSpec</span><span class=p>(</span><span class=s1>'gc-timer'</span><span class=p>,</span> <span class=n>TimeDomain</span><span class=o>.</span><span class=n>WATERMARK</span><span class=p>)</span>
-
- <span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
- <span class=n>element</span><span class=p>,</span>
+
+ <span class=k>def</span> <span class=nf>process</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
+ <span class=n>element</span><span class=p>,</span>
<span class=n>t</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>TimestampParam</span><span class=p>,</span>
- <span class=n>state</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>),</span>
+ <span class=n>state</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>),</span>
<span class=n>max_timestamp</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>MAX_TIMESTAMP</span><span class=p>),</span>
<span class=n>timer</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>TimerParam</span><span class=p>(</span><span class=n>TIMER</span><span class=p>)):</span>
<span class=n>update_state</span><span class=p>(</span><span class=n>state</span><span class=p>,</span> <span class=n>element</span><span class=p>)</span>
<span class=n>max_timestamp</span><span class=o>.</span><span class=n>add</span><span class=p>(</span><span class=n>t</span><span class=o>.</span><span class=n>micros</span><span class=p>)</span>
-
- <span class=c1># Set the timer to be one hour after the maximum timestamp seen. This will keep overwriting the same timer, so </span>
+
+ <span class=c1># Set the timer to be one hour after the maximum timestamp seen. This will keep overwriting the same timer, so</span>
<span class=c1># as long as there is activity on this key the state will stay active. Once the key goes inactive for one hour's</span>
<span class=c1># worth of event time (as measured by the watermark), then the gc timer will fire.</span>
<span class=n>expiration_time</span> <span class=o>=</span> <span class=n>Timestamp</span><span class=p>(</span><span class=n>micros</span><span class=o>=</span><span class=n>max_timestamp</span><span class=o>.</span><span class=n>read</span><span class=p>())</span> <span class=o>+</span> <span class=n>Duration</span><span class=p>(</span><span class=n>seconds</span><span class=o>=</span><span class=mi>60</span><span class=o>*</span><span class=mi>60</span><span class=p>)</span>
<span class=n>timer</span><span class=o>.</span><span class=n>set</span><span class=p>(</span><span class=n>expiration_time</span><span class=p>)</span>
-
+
<span class=nd>@on_timer</span><span class=p>(</span><span class=n>TIMER</span><span class=p>)</span>
- <span class=k>def</span> <span class=nf>expiry_callback</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
+ <span class=k>def</span> <span class=nf>expiry_callback</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span>
<span class=n>state</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>ALL_ELEMENTS</span><span class=p>),</span>
<span class=n>max_timestamp</span> <span class=o>=</span> <span class=n>DoFn</span><span class=o>.</span><span class=n>StateParam</span><span class=p>(</span><span class=n>MAX_TIMESTAMP</span><span class=p>)):</span>
<span class=n>state</span><span class=o>.</span><span class=n>clear</span><span class=p>()</span>
<span class=n>max_timestamp</span><span class=o>.</span><span class=n>clear</span><span class=p>()</span>
-
-
+
+
<span class=n>_</span> <span class=o>=</span> <span class=p>(</span><span class=n>p</span> <span class=o>|</span> <span class=s1>'Read per user'</span> <span class=o>>></span> <span class=n>ReadPerUser</span><span class=p>()</span>
<span class=o>|</span> <span class=s1>'User DoFn'</span> <span class=o>>></span> <span class=n>beam</span><span class=o>.</span><span class=n>ParDo</span><span class=p>(</span><span class=n>UserDoFn</span><span class=p>()))</span></code></pre></div></div><h3 id=state-timers-examples>11.5 State and timers examples</h3><p>Following are some example uses of state and timers</p><h4 id=joining-clicks-and-views>11.5.1. Joining clicks and views</h4><p>In this example, the p [...]
a stream of views, representing suggested product links displayed to the user on the home page, and a stream of
@@ -2628,7 +2626,7 @@ will give up on this join. While every click event should have a view event, som
lost and never make it to the Beam pipeline; the pipeline will similarly wait one hour after seeing a click event, and
give up if the view event does not arrive in that time. Input events are not ordered - it is possible to see the click
event before the view event. The one hour join timeout should be based on event time, not on processing time.</p><div class=language-java><div class=highlight><pre class=chroma><code class=language-java data-lang=java><span class=c1>// Read the event stream and key it by the link id.
-</span><span class=c1></span><span class=n>PCollection</span><span class=o><</span><span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>Event</span><span class=o>>></span> <span class=n>eventsPerLinkId</span> <span class=o>=</span>
+</span><span class=c1></span><span class=n>PCollection</span><span class=o><</span><span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>Event</span><span class=o>>></span> <span class=n>eventsPerLinkId</span> <span class=o>=</span>
<span class=n>readEvents</span><span class=o>()</span>
<span class=o>.</span><span class=na>apply</span><span class=o>(</span><span class=n>WithKeys</span><span class=o>.</span><span class=na>of</span><span class=o>(</span><span class=n>Event</span><span class=o>::</span><span class=n>getLinkId</span><span class=o>).</span><span class=na>withKeyType</span><span class=o>(</span><span class=n>TypeDescriptors</span><span class=o>.</span><span class=na>strings</span><span class=o>()));</span>
@@ -2639,7 +2637,7 @@ event before the view event. The one hour join timeout should be based on event
</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"click"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>ValueState</span><span class=o><</span><span class=n>Event</span><span class=o>>></span> <span class=n>clickState</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>va [...]
<span class=c1>// The maximum element timestamp seen so far.
-</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"maxTimestampSeen"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>CombiningState</span><span class=o><</span><span class=n>Long</span><span class=o>,</span> <span class=kt>long</span><span class=o>[],</span> <span class=n>Long</span><span class=o>>></span>
+</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"maxTimestampSeen"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>CombiningState</span><span class=o><</span><span class=n>Long</span><span class=o>,</span> <span class=kt>long</span><span class=o>[],</span> <span class=n>Long</span><span class=o>>></span>
<span class=n>maxTimestamp</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span class=na>combining</span><span class=o>(</span><span class=n>Max</span><span class=o>.</span><span class=na>ofLongs</span><span class=o>());</span>
<span class=c1>// Timer that fires when an hour goes by with an incomplete join.
@@ -2652,12 +2650,12 @@ event before the view event. The one hour join timeout should be based on event
<span class=nd>@AlwaysFetched</span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"click"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Event</span><span class=o>></span> <span class=n>clickState</span><span class=o>,</span>
<span class=nd>@AlwaysFetched</span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"maxTimestampSeen"</span><span class=o>)</span> <span class=n>CombiningState</span><span class=o><</span><span class=n>Long</span><span class=o>,</span> <span class=kt>long</span><span class=o>[],</span> <span class=n>Long</span><span class=o>></span> <span class=n>maxTimestampState</span><span class=o>,</span>
<span class=nd>@TimerId</span><span class=o>(</span><span class=s>"gcTimer"</span><span class=o>)</span> <span class=n>gcTimer</span><span class=o>,</span>
- <span class=n>OutputReceiver</span><span class=o><</span><span class=n>JoinedEvent</span><span class=o>></span> <span class=n>output</span><span class=o>)</span> <span class=o>{</span>
+ <span class=n>OutputReceiver</span><span class=o><</span><span class=n>JoinedEvent</span><span class=o>></span> <span class=n>output</span><span class=o>)</span> <span class=o>{</span>
<span class=c1>// Store the event into the correct state variable.
</span><span class=c1></span> <span class=n>Event</span> <span class=n>event</span> <span class=o>=</span> <span class=n>element</span><span class=o>.</span><span class=na>getValue</span><span class=o>();</span>
<span class=n>ValueState</span><span class=o><</span><span class=n>Event</span><span class=o>></span> <span class=n>valueState</span> <span class=o>=</span> <span class=n>event</span><span class=o>.</span><span class=na>getType</span><span class=o>().</span><span class=na>equals</span><span class=o>(</span><span class=n>VIEW</span><span class=o>)</span> <span class=o>?</span> <span class=n>viewState</span> <span class=o>:</span> <span class=n>clickState</span><span class=o>;</span>
<span class=n>valueState</span><span class=o>.</span><span class=na>write</span><span class=o>(</span><span class=n>event</span><span class=o>);</span>
-
+
<span class=n>Event</span> <span class=n>view</span> <span class=o>=</span> <span class=n>viewState</span><span class=o>.</span><span class=na>read</span><span class=o>();</span>
<span class=n>Event</span> <span class=n>click</span> <span class=o>=</span> <span class=n>clickState</span><span class=o>.</span><span class=na>read</span><span class=o>();</span>
<span class=o>(</span><span class=k>if</span> <span class=n>view</span> <span class=o>!=</span> <span class=kc>null</span> <span class=o>&&</span> <span class=n>click</span> <span class=o>!=</span> <span class=kc>null</span><span class=o>)</span> <span class=o>{</span>
@@ -2666,7 +2664,7 @@ event before the view event. The one hour join timeout should be based on event
<span class=n>clearState</span><span class=o>(</span><span class=n>viewState</span><span class=o>,</span> <span class=n>clickState</span><span class=o>,</span> <span class=n>maxTimestampState</span><span class=o>);</span>
<span class=o>}</span> <span class=k>else</span> <span class=o>{</span>
<span class=c1>// We've only seen on half of the join.
-</span><span class=c1></span> <span class=c1>// Set the timer to be one hour after the maximum timestamp seen. This will keep overwriting the same timer, so
+</span><span class=c1></span> <span class=c1>// Set the timer to be one hour after the maximum timestamp seen. This will keep overwriting the same timer, so
</span><span class=c1></span> <span class=c1>// as long as there is activity on this key the state will stay active. Once the key goes inactive for one hour's
</span><span class=c1></span> <span class=c1>// worth of event time (as measured by the watermark), then the gc timer will fire.
</span><span class=c1></span> <span class=n>maxTimestampState</span><span class=o>.</span><span class=na>add</span><span class=o>(</span><span class=n>ts</span><span class=o>.</span><span class=na>getMillis</span><span class=o>());</span>
@@ -2682,7 +2680,7 @@ event before the view event. The one hour join timeout should be based on event
<span class=c1>// An hour has gone by with an incomplete join. Give up and clear the state.
</span><span class=c1></span> <span class=n>clearState</span><span class=o>(</span><span class=n>viewState</span><span class=o>,</span> <span class=n>clickState</span><span class=o>,</span> <span class=n>maxTimestampState</span><span class=o>);</span>
<span class=o>}</span>
-
+
<span class=kd>private</span> <span class=kt>void</span> <span class=nf>clearState</span><span class=o>(</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"view"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Event</span><span class=o>></span> <span class=n>viewState</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"click"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Event</span><span class=o>></span> <span class=n>clickState</span><span class=o>,</span>
@@ -2701,9 +2699,9 @@ we want to batch ten seconds worth of events together in order to reduce the num
</span><span class=c1></span> <span class=nd>@StateId</span><span class=o>(</span><span class=s>"isTimerSet"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>StateSpec</span><span class=o><</span><span class=n>ValueState</span><span class=o><</span><span class=n>Boolean</span><span class=o>>></span> <span class=n>isTimerSet</span> <span class=o>=</span> <span class=n>StateSpecs</span><span class=o>.</span><span clas [...]
<span class=c1>// The processing-time timer user to publish the RPC.
</span><span class=c1></span> <span class=nd>@TimerId</span><span class=o>(</span><span class=s>"outputState"</span><span class=o>)</span> <span class=kd>private</span> <span class=kd>final</span> <span class=n>TimerSpec</span> <span class=n>timer</span> <span class=o>=</span> <span class=n>TimerSpecs</span><span class=o>.</span><span class=na>timer</span><span class=o>(</span><span class=n>TimeDomain</span><span class=o>.</span><span class=na>PROCESSING_TIME</span><span class=o [...]
-
+
<span class=nd>@ProcessElement</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>process</span><span class=o>(</span>
- <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
+ <span class=nd>@Element</span> <span class=n>KV</span><span class=o><</span><span class=n>String</span><span class=o>,</span> <span class=n>ValueT</span><span class=o>></span> <span class=n>element</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=n>BagState</span><span class=o><</span><span class=n>ValueT</span><span class=o>></span> <span class=n>elementsState</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"isTimerSet"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Boolean</span><span class=o>></span> <span class=n>isTimerSetState</span><span class=o>,</span>
<span class=nd>@TimerId</span><span class=o>(</span><span class=s>"outputState"</span><span class=o>)</span> <span class=n>Timer</span> <span class=n>timer</span><span class=o>)</span> <span class=o>{</span>
@@ -2715,7 +2713,7 @@ we want to batch ten seconds worth of events together in order to reduce the num
<span class=n>isTimerSetState</span><span class=o>.</span><span class=na>write</span><span class=o>(</span><span class=kc>true</span><span class=o>);</span>
<span class=o>}</span>
<span class=o>}</span>
-
+
<span class=nd>@OnTimer</span><span class=o>(</span><span class=s>"outputState"</span><span class=o>)</span> <span class=kd>public</span> <span class=kt>void</span> <span class=nf>onTimer</span><span class=o>(</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"state"</span><span class=o>)</span> <span class=n>BagState</span><span class=o><</span><span class=n>ValueT</span><span class=o>></span> <span class=n>elementsState</span><span class=o>,</span>
<span class=nd>@StateId</span><span class=o>(</span><span class=s>"isTimerSet"</span><span class=o>)</span> <span class=n>ValueState</span><span class=o><</span><span class=n>Boolean</span><span class=o>></span> <span class=n>isTimerSetState</span><span class=o>)</span> <span class=o>{</span>
diff --git a/website/generated-content/documentation/runners/jet/index.html b/website/generated-content/documentation/runners/jet/index.html
index 9af282a..79e6a28 100644
--- a/website/generated-content/documentation/runners/jet/index.html
+++ b/website/generated-content/documentation/runners/jet/index.html
@@ -31,13 +31,13 @@ $ bin/jet-start &</code></pre></div><p>Check the cluster is up and running:<
Version: 3.0
Size: 2
-ADDRESS UUID
+ADDRESS UUID
[192.168.0.117]:5701 76bea7ba-f032-4c25-ad04-bdef6782f481
[192.168.0.117]:5702 03ecfaa2-be16-41b6-b5cf-eea584d7fb86</code></pre></div><div class=version-jet4><pre><code>State: ACTIVE
Version: 4.0
Size: 2
-ADDRESS UUID
+ADDRESS UUID
[192.168.0.117]:5701 b9937bba-32aa-48ba-8e32-423aafed763b
[192.168.0.117]:5702 dfeadfb2-3ba5-4d1c-95e7-71a1a3ca4937</code></pre></div><p>Change directory to the Beam Examples project and issue following command to submit and execute your
Pipeline on the remote Jet cluster.
diff --git a/website/generated-content/documentation/runners/samza/index.html b/website/generated-content/documentation/runners/samza/index.html
index 9c1236a..ee06420 100644
--- a/website/generated-content/documentation/runners/samza/index.html
+++ b/website/generated-content/documentation/runners/samza/index.html
@@ -40,8 +40,7 @@
<span class=o><</span><span class=n>artifactId</span><span class=o>></span><span class=n>samza</span><span class=o>-</span><span class=n>kv</span><span class=o>-</span><span class=n>rocksdb_2</span><span class=o>.</span><span class=na>11</span><span class=o></</span><span class=n>artifactId</span><span class=o>></span>
<span class=o><</span><span class=n>version</span><span class=o>></span><span class=n>$</span><span class=o>{</span><span class=n>samza</span><span class=o>.</span><span class=na>version</span><span class=o>}</</span><span class=n>version</span><span class=o>></span>
<span class=o><</span><span class=n>scope</span><span class=o>></span><span class=n>runtime</span><span class=o></</span><span class=n>scope</span><span class=o>></span>
-<span class=o></</span><span class=n>dependency</span><span class=o>></span>
- </code></pre></div></div></p><h2 id=executing-a-pipeline-with-samza-runner>Executing a pipeline with Samza Runner</h2><p>If you run your pipeline locally or deploy it to a standalone cluster with all the jars and resource files, no packaging is required. For example, the following command runs the WordCount example:</p><pre><code>$ mvn exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount \
+<span class=o></</span><span class=n>dependency</span><span class=o>></span></code></pre></div></div></p><h2 id=executing-a-pipeline-with-samza-runner>Executing a pipeline with Samza Runner</h2><p>If you run your pipeline locally or deploy it to a standalone cluster with all the jars and resource files, no packaging is required. For example, the following command runs the WordCount example:</p><pre><code>$ mvn exec:java -Dexec.mainClass=org.apache.beam.examples.WordCount \
-Psamza-runner \
-Dexec.args="--runner=SamzaRunner \
--inputFile=/path/to/input \
diff --git a/website/generated-content/documentation/sdks/java-thirdparty/index.html b/website/generated-content/documentation/sdks/java-thirdparty/index.html
index d84bc9f..9238504 100644
--- a/website/generated-content/documentation/sdks/java-thirdparty/index.html
+++ b/website/generated-content/documentation/sdks/java-thirdparty/index.html
@@ -17,7 +17,7 @@ This parser is simply initialized with this schema and the list of fields you wa
@Setup
public void setup() throws NoSuchMethodException {
- parser = new HttpdLoglineParser<>(WebEvent.class,
+ parser = new HttpdLoglineParser<>(WebEvent.class,
"%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\"");
parser.addParseTarget("setIP", "IP:connection.client.host");
parser.addParseTarget("setQueryImg", "STRING:request.firstline.uri.query.img");
diff --git a/website/generated-content/documentation/sdks/java/testing/nexmark/index.html b/website/generated-content/documentation/sdks/java/testing/nexmark/index.html
index fbcb727..7c1a82d 100644
--- a/website/generated-content/documentation/sdks/java/testing/nexmark/index.html
+++ b/website/generated-content/documentation/sdks/java/testing/nexmark/index.html
@@ -77,7 +77,7 @@ or may be published to Pub/Sub or Kafka.</p><p>The query results may be:</p><ul>
</code></pre><h3 id=spark-runner-specific-configuration>Spark runner specific configuration</h3><pre><code>--manageResources=false --monitorJobs=true \
--sparkMaster=local \
-Dspark.ui.enabled=false -DSPARK_LOCAL_IP=localhost -Dsun.io.serialization.extendedDebugInfo=true
-</code></pre><h3 id=kafka-sourcesink-configuration-parameters>Kafka source/sink configuration parameters</h3><p>Set Kafka host/ip (for example, “localhost:9092”):</p><pre><code>--bootstrapServers=<kafka host/ip>
+</code></pre><h3 id=kafka-sourcesink-configuration-parameters>Kafka source/sink configuration parameters</h3><p>Set Kafka host/ip (for example, “localhost:9092”):</p><pre><code>--bootstrapServers=<kafka host/ip>
</code></pre><p>Write results into Kafka topic:</p><pre><code>--sinkType=KAFKA
</code></pre><p>Set topic name which will be used for benchmark results:</p><pre><code>--kafkaResultsTopic=<topic name>
</code></pre><p>Write or/and read events into/from Kafka topic:</p><pre><code>--sourceType=KAFKA
diff --git a/website/generated-content/documentation/sdks/python-type-safety/index.html b/website/generated-content/documentation/sdks/python-type-safety/index.html
index d681bb3..cc46914 100644
--- a/website/generated-content/documentation/sdks/python-type-safety/index.html
+++ b/website/generated-content/documentation/sdks/python-type-safety/index.html
@@ -1,7 +1,7 @@
<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><title>Ensuring Python Type Safety</title><meta name=description content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and D [...]
<span class=sr-only>Toggle navigation</span>
<span class=icon-bar></span><span class=icon-bar></span><span class=icon-bar></span></button>
-<a href=/ class=navbar-brand><img alt=Brand style=height:25px src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask closed"></div><div id=navbar class="navbar-container closed"><ul class="nav navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a href=/documentation/>Documentation</a></li><li><a href=/documentation/sdks/java/>Languages</a></li><li><a href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a href=/roadmap/>Roadmap</a></li>< [...]
+<a href=/ class=navbar-brand><img alt=Brand style=height:25px src=/images/beam_logo_navbar.png></a></div><div class="navbar-mask closed"></div><div id=navbar class="navbar-container closed"><ul class="nav navbar-nav"><li><a href=/get-started/beam-overview/>Get Started</a></li><li><a href=/documentation/>Documentation</a></li><li><a href=/documentation/sdks/java/>Languages</a></li><li><a href=/documentation/runners/capability-matrix/>RUNNERS</a></li><li><a href=/roadmap/>Roadmap</a></li>< [...]
The Beam SDK for Python implements a subset of <a href=https://www.python.org/dev/peps/pep-0484/>PEP 484</a> and aims to follow it as closely as possible in its own typehints module.</p><p>These flags control Beam type safety:</p><ul><li><p><code>--no_pipeline_type_check</code></p><p>Disables type checking during pipeline construction.
Default is to perform these checks.</p></li><li><p><code>--runtime_type_check</code></p><p>Enables runtime type checking of every element.
This may affect pipeline performance, so the default is to skip these checks.</p></li></ul><h2 id=benefits-of-type-hints>Benefits of Type Hints</h2><p>When you use type hints, Beam raises exceptions during pipeline construction time, rather than runtime.
@@ -73,7 +73,8 @@ If the input to <code>MyTransform</code> is of type <code>str</code>, Beam will
It also supports Python’s typing module types, which are internally converted to Beam internal types.</p><blockquote><p>For new code, it is recommended to use <a href=https://docs.python.org/3/library/typing.html><strong>typing</strong></a> module types.</p></blockquote><h3 id=simple-type-hints>Simple Type Hints</h3><p>Type hints can be of any class, from <code>int</code> and <code>str</code>, to user-defined classes. If you have a class as a type hint, you may want to define a cod [...]
<span class=n>p</span> <span class=o>|</span> <span class=n>beam</span><span class=o>.</span><span class=n>Create</span><span class=p>([</span><span class=s1>'a'</span><span class=p>])</span> <span class=o>|</span> <span class=n>beam</span><span class=o>.</span><span class=n>Map</span><span class=p>(</span><span class=k>lambda</span> <span class=n>x</span><span class=p>:</span> <span class=mi>3</span><span class=p>)</span><span class=o>.</span><span class=n>with_output_types</spa [...]
<span class=n>p</span> <span class=o>|</span> <span class=n>beam</span><span class=o>.</span><span class=n>Create</span><span class=p>([</span><span class=s1>'a'</span><span class=p>])</span> <span class=o>|</span> <span class=n>beam</span><span class=o>.</span><span class=n>Map</span><span class=p>(</span><span class=k>lambda</span> <span class=n>x</span><span class=p>:</span> <span class=mi>3</span><span class=p>)</span><span class=o>.</span><span class=n>with_output_types</spa [...]
-<span class=n>p</span><span class=o>.</span><span class=n>run</span><span class=p>()</span></code></pre></div></div><p>Note that because runtime type checks are done for each <code>PCollection</code> element, enabling this feature may incur a significant performance penalty. It is therefore recommended that runtime type checks are disabled for production pipelines.</p><h2 id=use-of-type-hints-in-coders>Use of Type Hints in Coders</h2><p>When your pipeline reads, writes, or otherwise mate [...]
+<span class=n>p</span><span class=o>.</span><span class=n>run</span><span class=p>()</span></code></pre></div></div><p>Note that because runtime type checks are done for each <code>PCollection</code> element, enabling this feature may incur a significant performance penalty. It is therefore recommended that runtime type checks are disabled for production pipelines. See the following section for a quicker, production-friendly alternative.</p><h3 id=faster-runtime-type-checking>Faster Runt [...]
+These are planned to be supported in a future release of Beam.</p><h2 id=use-of-type-hints-in-coders>Use of Type Hints in Coders</h2><p>When your pipeline reads, writes, or otherwise materializes its data, the elements in your <code>PCollection</code> need to be encoded and decoded to and from byte strings. Byte strings are used for intermediate storage, for comparing keys in <code>GroupByKey</code> operations, and for reading from sources and writing to sinks.</p><p>The Beam SDK for Pyt [...]
<span class=k>class</span> <span class=nc>Player</span><span class=p>(</span><span class=nb>object</span><span class=p>):</span>
<span class=k>def</span> <span class=fm>__init__</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=n>team</span><span class=p>,</span> <span class=n>name</span><span class=p>):</span>
diff --git a/website/generated-content/feed.xml b/website/generated-content/feed.xml
index 6ef9169..18053c6 100644
--- a/website/generated-content/feed.xml
+++ b/website/generated-content/feed.xml
@@ -1,4 +1,244 @@
-<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0"><channel><title>Apache Beam</title><description>Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number [...]
+<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0"><channel><title>Apache Beam</title><description>Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number [...]
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>The importance of static type checking in a dynamically
+typed language like Python is not up for debate. Type hints
+allow developers to leverage a strong typing system to:</p>
+<ul>
+<li>write better code,</li>
+<li>self-document ambiguous programming logic, and</li>
+<li>inform intelligent code completion in IDEs like PyCharm.</li>
+</ul>
+<p>This is why we&rsquo;re excited to announce upcoming improvements to
+the <code>typehints</code> module of Beam&rsquo;s Python SDK, including support
+for typed PCollections and Python 3 style annotations on PTransforms.</p>
+<h1 id="improved-annotations">Improved Annotations</h1>
+<p>Today, you have the option to declare type hints on PTransforms using either
+class decorators or inline functions.</p>
+<p>For instance, a PTransform with decorated type hints might look like this:</p>
+<pre><code>@beam.typehints.with_input_types(int)
+@beam.typehints.with_output_types(str)
+class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>Using inline functions instead, the same transform would look like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll):
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr()).with_input_types(int).with_output_types(str)
+</code></pre><p>Both methods have problems. Class decorators are syntax-heavy,
+requiring two additional lines of code, whereas inline functions provide type hints
+that aren&rsquo;t reusable across other instances of the same transform. Additionally, both
+methods are incompatible with static type checkers like MyPy.</p>
+<p>With Python 3 annotations however, we can subvert these problems to provide a
+clean and reusable type hint experience. Our previous transform now looks like this:</p>
+<pre><code>class IntToStr(beam.PTransform):
+def expand(self, pcoll: PCollection[int]) -&gt; PCollection[str]:
+return pcoll | beam.Map(lambda num: str(num))
+strings = numbers | beam.ParDo(IntToStr())
+</code></pre><p>These type hints will actively hook into the internal Beam typing system to
+play a role in pipeline type checking, and runtime type checking.</p>
+<p>So how does this work?</p>
+<h2 id="typed-pcollections">Typed PCollections</h2>
+<p>You guessed it! The PCollection class inherits from <code>typing.Generic</code>, allowing it to be
+parameterized with either zero types (denoted <code>PCollection</code>) or one type (denoted <code>PCollection[T]</code>).</p>
+<ul>
+<li>A PCollection with zero types is implicitly converted to <code>PCollection[Any]</code>.</li>
+<li>A PCollection with one type can have any nested type (e.g. <code>Union[int, str]</code>).</li>
+</ul>
+<p>Internally, Beam&rsquo;s typing system makes these annotations compatible with other
+type hints by removing the outer PCollection container.</p>
+<h2 id="pbegin-pdone-none">PBegin, PDone, None</h2>
+<p>Finally, besides PCollection, a valid annotation on the <code>expand(...)</code> method of a PTransform is
+<code>PBegin</code> or <code>None</code>. These are generally used for PTransforms that begin or end with an I/O operation.</p>
+<p>For instance, when saving data, your transform&rsquo;s output type should be <code>None</code>.</p>
+<pre><code>class SaveResults(beam.PTransform):
+def expand(self, pcoll: PCollection[str]) -&gt; None:
+return pcoll | beam.io.WriteToBigQuery(...)
+</code></pre><h1 id="next-steps">Next Steps</h1>
+<p>What are you waiting for.. start using annotations on your transforms!</p>
+<p>For more background on type hints in Python, see:
+<a href="https://beam.apache.org/documentation/sdks/python-type-safety/">Ensuring Python Type Safety</a>.</p>
+<p>Finally, please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description><link>/blog/python-improved-annotations/</link><pubDate>Fri, 21 Aug 2020 00:00:01 -0800</pubDate><guid>/blog/python-improved-annotations/</guid><category>blog</category><category>python</category><category>typing</category></item><item><title>Performance-Driven Runtime Type Checking for the Python SDK</title><description>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p>In this blog post, we&rsquo;re announcing the upcoming release of a new, opt-in
+runtime type checking system for Beam&rsquo;s Python SDK that&rsquo;s optimized for performance
+in both development and production environments.</p>
+<p>But let&rsquo;s take a step back - why do we even care about runtime type checking
+in the first place? Let&rsquo;s look at an example.</p>
+<pre><code>class MultiplyNumberByTwo(beam.DoFn):
+def process(self, element: int):
+return element * 2
+p = Pipeline()
+p | beam.Create(['1', '2'] | beam.ParDo(MultiplyNumberByTwo())
+</code></pre><p>In this code, we passed a list of strings to a DoFn that&rsquo;s clearly intended for use with
+integers. Luckily, this code will throw an error during pipeline construction because
+the inferred output type of <code>beam.Create(['1', '2'])</code> is <code>str</code> which is incompatible with
+the declared input type of <code>MultiplyNumberByTwo.process</code> which is <code>int</code>.</p>
+<p>However, what if we turned pipeline type checking off using the <code>no_pipeline_type_check</code>
+flag? Or more realistically, what if the input PCollection to <code>MultiplyNumberByTwo</code> arrived
+from a database, meaning that the output data type can only be known at runtime?</p>
+<p>In either case, no error would be thrown during pipeline construction.
+And even at runtime, this code works. Each string would be multiplied by 2,
+yielding a result of <code>['11', '22']</code>, but that&rsquo;s certainly not the outcome we want.</p>
+<p>So how do you debug this breed of &ldquo;hidden&rdquo; errors? More broadly speaking, how do you debug
+any typing or serialization error in Beam?</p>
+<p>The answer is to use runtime type checking.</p>
+<h1 id="runtime-type-checking-rtc">Runtime Type Checking (RTC)</h1>
+<p>This feature works by checking that actual input and output values satisfy the declared
+type constraints during pipeline execution. If you ran the code from before with
+<code>runtime_type_check</code> on, you would receive the following error message:</p>
+<pre><code>Type hint violation for 'ParDo(MultiplyByTwo)': requires &lt;class 'int'&gt; but got &lt;class 'str'&gt; for element
+</code></pre><p>This is an actionable error message - it tells you that either your code has a bug
+or that your declared type hints are incorrect. Sounds simple enough, so what&rsquo;s the catch?</p>
+<p><em>It is soooo slowwwwww.</em> See for yourself.</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal Pipeline</th>
+<th>Runtime Type Checking Pipeline</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+</tr>
+</tbody>
+</table>
+<p>In this micro-benchmark, the pipeline with runtime type checking was over 10x slower,
+with the gap only increasing as our input PCollection increased in size.</p>
+<p>So, is there any production-friendly alternative?</p>
+<h1 id="performance-runtime-type-check">Performance Runtime Type Check</h1>
+<p>There is! We developed a new flag called <code>performance_runtime_type_check</code> that
+minimizes its footprint on the pipeline&rsquo;s time complexity using a combination of</p>
+<ul>
+<li>efficient Cython code,</li>
+<li>smart sampling techniques, and</li>
+<li>optimized mega type-hints.</li>
+</ul>
+<p>So what do the new numbers look like?</p>
+<table>
+<thead>
+<tr>
+<th>Element Size</th>
+<th>Normal</th>
+<th>RTC</th>
+<th>Performance RTC</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>5.3 sec</td>
+<td>5.6 sec</td>
+<td>5.4 sec</td>
+</tr>
+<tr>
+<td>2,001</td>
+<td>9.4 sec</td>
+<td>57.2 sec</td>
+<td>11.2 sec</td>
+</tr>
+<tr>
+<td>10,001</td>
+<td>24.5 sec</td>
+<td>259.8 sec</td>
+<td>25.5 sec</td>
+</tr>
+<tr>
+<td>18,001</td>
+<td>38.7 sec</td>
+<td>450.5 sec</td>
+<td>39.4 sec</td>
+</tr>
+</tbody>
+</table>
+<p>On average, the new Performance RTC is 4.4% slower than a normal pipeline whereas the old RTC
+is over 900% slower! Additionally, as the size of the input PCollection increases, the fixed cost
+of setting up the Performance RTC system is spread across each element, decreasing the relative
+impact on the overall pipeline. With 18,001 elements, the difference is less than 1 second.</p>
+<h2 id="how-does-it-work">How does it work?</h2>
+<p>There are three key factors responsible for this upgrade in performance.</p>
+<ol>
+<li>
+<p>Instead of type checking all values, we only type check a subset of values, known as
+a sample in statistics. Initially, we sample a substantial number of elements, but as our
+confidence that the element type won&rsquo;t change over time increases, we reduce our
+sampling rate (up to a fixed minimum).</p>
+</li>
+<li>
+<p>Whereas the old RTC system used heavy wrappers to perform the type check, the new RTC system
+moves the type check to a Cython-optimized, non-decorated portion of the codebase. For reference,
+Cython is a programming language that gives C-like performance to Python code.</p>
+</li>
+<li>
+<p>Finally, we use a single mega type hint to type-check only the output values of transforms
+instead of type-checking both the input and output values separately. This mega typehint is composed of
+the original transform&rsquo;s output type constraints along with all consumer transforms&rsquo; input type
+constraints. Using this mega type hint allows us to reduce overhead while simultaneously allowing
+us to throw <em>more actionable errors</em>. For instance, consider the following error (which was
+generated from the old RTC system):</p>
+</li>
+</ol>
+<pre><code>Runtime type violation detected within ParDo(DownstreamDoFn): Type-hint for argument: 'element' violated. Expected an instance of &lt;class ‘str’&gt;, instead found 9, an instance of &lt;class ‘int’&gt;.
+</code></pre><p>This error tells us that the <code>DownstreamDoFn</code> received an <code>int</code> when it was expecting a <code>str</code>, but doesn&rsquo;t tell us
+who created that <code>int</code> in the first place. Who is the offending upstream transform that&rsquo;s responsible for
+this <code>int</code>? Presumably, <em>that</em> transform&rsquo;s output type hints were too expansive (e.g. <code>Any</code>) or otherwise non-existent because
+no error was thrown during the runtime type check of its output.</p>
+<p>The problem here boils down to a lack of context. If we knew who our consumers were when type
+checking our output, we could simultaneously type check our output value against our output type
+constraints and every consumers&rsquo; input type constraints to know whether there is <em>any</em> possibility
+for a mismatch. This is exactly what the mega type hint does, and it allows us to throw errors
+at the point of declaration rather than the point of exception, saving you valuable time
+while providing higher quality error messages.</p>
+<p>So what would the same error look like using Performance RTC? It&rsquo;s the exact same string but with one additional line:</p>
+<pre><code>[while running 'ParDo(UpstreamDoFn)']
+</code></pre><p>And that&rsquo;s much more actionable for an investigation :)</p>
+<h1 id="next-steps">Next Steps</h1>
+<p>Go play with the new <code>performance_runtime_type_check</code> feature!</p>
+<p>It&rsquo;s in an experimental state so please
+<a href="https://beam.apache.org/community/contact-us/">let us know</a>
+if you encounter any issues.</p></description><link>/blog/python-performance-runtime-type-checking/</link><pubDate>Fri, 21 Aug 2020 00:00:01 -0800</pubDate><guid>/blog/python-performance-runtime-type-checking/</guid><category>blog</category><category>python</category><category>typing</category></item><item><title>Apache Beam 2.23.0</title><description>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -513,189 +753,4 @@ See the <a href="/get-started/downloads/#2190-2020-02-04">download page</a
</ul>
<h2 id="list-of-contributors">List of Contributors</h2>
<p>According to git shortlog, the following people contributed to the 2.19.0 release. Thank you to all contributors!</p>
-<p>Ahmet Altay, Alex Amato, Alexey Romanenko, Andrew Pilloud, Ankur Goenka, Anton Kedin, Boyuan Zhang, Brian Hulette, Brian Martin, Chamikara Jayalath, Charles Chen, Craig Chambers, Daniel Oliveira, David Moravek, David Rieber, Dustin Rhodes, Etienne Chauchot, Gleb Kanterov, Hai Lu, Heejong Lee, Ismaël Mejía, Jan Lukavský, Jason Kuster, Jean-Baptiste Onofré, Jeff Klukas, João Cabrita, J Ross Thomson, Juan Rael, Juta, Kasia Kucharczyk, Kengo Seki, Kenneth Jung, Kenneth Knowles, Kyle We [...]
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<p>We are happy to present the new 2.18.0 release of Beam. This release includes both improvements and new functionality.
-See the <a href="/get-started/downloads/#2180-2020-01-23">download page</a> for this release.</p>
-<p>For more information on changes in 2.18.0, check out the
-<a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12346383&amp;projectId=12319527">detailed release notes</a>.</p>
-<h2 id="highlights">Highlights</h2>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8470">BEAM-8470</a> - Create a new Spark runner based on Spark Structured streaming framework</li>
-</ul>
-<h3 id="ios">I/Os</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7636">BEAM-7636</a> - Added SqsIO v2 support.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8513">BEAM-8513</a> - RabbitMqIO: Allow reads from exchange-bound queue without declaring the exchange.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8540">BEAM-8540</a> - Fix CSVSink example in FileIO docs</li>
-</ul>
-<h3 id="new-features--improvements">New Features / Improvements</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-5878">BEAM-5878</a> - Added support DoFns with Keyword-only arguments in Python 3.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-6756">BEAM-6756</a> - Improved support for lazy iterables in schemas (Java).</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-4776">BEAM-4776</a> AND <a href="https://issues.apache.org/jira/browse/BEAM-4777">BEAM-4777</a> - Added metrics supports to portable runners.</li>
-<li>Various improvements to Interactive Beam: <a href="https://issues.apache.org/jira/browse/BEAM-7760">BEAM-7760</a>, <a href="https://issues.apache.org/jira/browse/BEAM-8379">BEAM-8379</a>, <a href="https://issues.apache.org/jira/browse/BEAM-8016">BEAM-8016</a>, <a href="https://issues.apache.org/jira/browse/BEAM-8016">BEAM-8016</a>.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8658">BEAM-8658</a> - Optionally set artifact staging port in FlinkUberJarJobServer.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8660">BEAM-8660</a> - Override returned artifact staging endpoint</li>
-</ul>
-<h3 id="sql">SQL</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8343">BEAM-8343</a> - [SQL] Add means for IO APIs to support predicate and/or project push-down when running SQL pipelines. And <a href="https://issues.apache.org/jira/browse/BEAM-8468">BEAM-8468</a>, <a href="https://issues.apache.org/jira/browse/BEAM-8365">BEAM-8365</a>, <a href="https://issues.apache.org/jira/browse/BEAM-8508">BEAM-8508</a>.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8427">BEAM-8427</a> - [SQL] Add support for MongoDB source.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8456">BEAM-8456</a> - Add pipeline option to control truncate of BigQuery data processed by Beam SQL.</li>
-</ul>
-<h3 id="breaking-changes">Breaking Changes</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8814">BEAM-8814</a> - &ndash;no_auth flag changed to boolean type.</li>
-</ul>
-<h3 id="deprecations">Deprecations</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8252">BEAM-8252</a> AND <a href="https://issues.apache.org/jira/browse/BEAM-8254">BEAM-8254</a> Add worker_region and worker_zone options. Deprecated &ndash;zone flag and &ndash;worker_region experiment argument.</li>
-</ul>
-<h3 id="dependency-changes">Dependency Changes</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7078">BEAM-7078</a> - com.amazonaws:amazon-kinesis-client updated to 1.13.0.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8822">BEAM-8822</a> - Upgrade Hadoop dependencies to version 2.8.</li>
-</ul>
-<h3 id="bugfixes">Bugfixes</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7917">BEAM-7917</a> - Python datastore v1new fails on retry.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7981">BEAM-7981</a> - ParDo function wrapper doesn&rsquo;t support Iterable output types.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8146">BEAM-8146</a> - SchemaCoder/RowCoder have no equals() function.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8347">BEAM-8347</a> - UnboundedRabbitMqReader can fail to advance watermark if no new data comes in.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8352">BEAM-8352</a> - Reading records in background may lead to OOM errors</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8480">BEAM-8480</a> - Explicitly set restriction coder for bounded reader wrapper SDF.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8515">BEAM-8515</a> - Ensure that ValueProvider types have equals/hashCode implemented for comparison reasons.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8579">BEAM-8579</a> - Strip UTF-8 BOM bytes (if present) in TextSource.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8657">BEAM-8657</a> - Not doing Combiner lifting for data-driven triggers.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8663">BEAM-8663</a> - BundleBasedRunner Stacked Bundles don&rsquo;t respect PaneInfo.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8667">BEAM-8667</a> - Data channel should to avoid unlimited buffering in Python SDK.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8802">BEAM-8802</a> - Timestamp combiner not respected across bundles in streaming mode.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8803">BEAM-8803</a> - Default behaviour for Python BQ Streaming inserts sink should be to retry always.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8825">BEAM-8825</a> - OOM when writing large numbers of &lsquo;narrow&rsquo; rows.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8835">BEAM-8835</a> - Artifact retrieval fails with FlinkUberJarJobServer</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8836">BEAM-8836</a> - ExternalTransform is not providing a unique name</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8884">BEAM-8884</a> - Python MongoDBIO TypeError when splitting.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-9041">BEAM-9041</a> - SchemaCoder equals should not rely on from/toRowFunction equality.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-9042">BEAM-9042</a> - AvroUtils.schemaCoder(schema) produces a not serializable SchemaCoder.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-9065">BEAM-9065</a> - Spark runner accumulates metrics (incorrectly) between runs.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-6303">BEAM-6303</a> - Add .parquet extension to files in ParquetIO.</li>
-<li>Various bug fixes and performance improvements.</li>
-</ul>
-<h3 id="known-issues">Known Issues</h3>
-<ul>
-<li>
-<p><a href="https://issues.apache.org/jira/browse/BEAM-8882">BEAM-8882</a> - Python: <code>beam.Create</code> no longer preserves order unless <code>reshuffle=False</code> is passed in as an argument.</p>
-<p>You may encounter this issue when using DirectRunner.</p>
-</li>
-<li>
-<p><a href="https://issues.apache.org/jira/browse/BEAM-9065">BEAM-9065</a> - Spark runner accumulates metrics (incorrectly) between runs</p>
-</li>
-<li>
-<p><a href="https://issues.apache.org/jira/browse/BEAM-9123">BEAM-9123</a> - HadoopResourceId returns wrong directory name</p>
-</li>
-<li>
-<p>See a full list of open <a href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20BEAM%20AND%20affectedVersion%20%3D%202.18.0%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC">issues that affect</a> this version.</p>
-</li>
-<li>
-<p><a href="https://issues.apache.org/jira/browse/BEAM-9144">BEAM-9144</a> - If you are using Avro 1.9.x with Beam you should not upgrade to this version. There is an issue with timestamp conversions. A fix will be available in the next release.</p>
-</li>
-</ul>
-<h2 id="list-of-contributors">List of Contributors</h2>
-<p>According to git shortlog, the following people contributed to the 2.18.0 release. Thank you to all contributors!</p>
-<p>Ahmet Altay, Aizhamal Nurmamat kyzy, Alan Myrvold, Alexey Romanenko, Alex Van Boxel, Andre Araujo, Andrew Crites, Andrew Pilloud, Aryan Naraghi, Boyuan Zhang, Brian Hulette, bumblebee-coming, Cerny Ondrej, Chad Dombrova, Chamikara Jayalath, Changming Ma, Chun Yang, cmachgodaddy, Colm O hEigeartaigh, Craig Chambers, Daniel Oliveira, Daniel Robert, David Cavazos, David Moravek, David Song, dependabot[bot], Derek, Dmytro Sadovnychyi, Elliotte Rusty Harold, Etienne Chauchot, Hai Lu, He [...]
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<p>We are happy to present the new 2.17.0 release of Beam. This release includes both improvements and new functionality.
-Users of the MongoDbIO connector are encouraged to upgrade to this release to address a <a href="/security/CVE-2020-1929/">security vulnerability</a>.</p>
-<p>See the <a href="/get-started/downloads/#2170-2020-01-06">download page</a> for this release.</p>
-<p>For more information on changes in 2.17.0, check out the
-<a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12345970&amp;projectId=12319527">detailed release notes</a>.</p>
-<h2 id="highlights">Highlights</h2>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7962">BEAM-7962</a> - Drop support for Flink 1.5 and 1.6</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7635">BEAM-7635</a> - Migrate SnsIO to AWS SDK for Java 2</li>
-<li>Improved usability for portable Flink Runner
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8183">BEAM-8183</a> - Optionally bundle multiple pipelines into a single Flink jar.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8372">BEAM-8372</a> - Allow submission of Flink UberJar directly to flink cluster.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8471">BEAM-8471</a> - Flink native job submission for portable pipelines.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8312">BEAM-8312</a> - Flink portable pipeline jars do not need to stage artifacts remotely.</li>
-</ul>
-</li>
-</ul>
-<h3 id="new-features--improvements">New Features / Improvements</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7730">BEAM-7730</a> - Add Flink 1.9 build target and Make FlinkRunner compatible with Flink 1.9.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7990">BEAM-7990</a> - Add ability to read parquet files into PCollection of pyarrow.Table.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8355">BEAM-8355</a> - Make BooleanCoder a standard coder.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8394">BEAM-8394</a> - Add withDataSourceConfiguration() method in JdbcIO.ReadRows class.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-5428">BEAM-5428</a> - Implement cross-bundle state caching.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-5967">BEAM-5967</a> - Add handling of DynamicMessage in ProtoCoder.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7473">BEAM-7473</a> - Update RestrictionTracker within Python to not be required to be thread safe.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-7920">BEAM-7920</a> - Added AvroTableProvider to Beam SQL.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8098">BEAM-8098</a> - Improve documentation on BigQueryIO.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8100">BEAM-8100</a> - Add exception handling to Json transforms in Java SDK.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8306">BEAM-8306</a> - Improve estimation of data byte size reading from source in ElasticsearchIO.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8351">BEAM-8351</a> - Support passing in arbitrary KV pairs to sdk worker via external environment config.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8396">BEAM-8396</a> - Default to LOOPBACK mode for local flink (spark, &hellip;) runner.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8410">BEAM-8410</a> - JdbcIO should support setConnectionInitSqls in its DataSource.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8609">BEAM-8609</a> - Add HllCount to Java transform catalog.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8861">BEAM-8861</a> - Disallow self-signed certificates by default in ElasticsearchIO.</li>
-</ul>
-<h3 id="dependency-changes">Dependency Changes</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8285">BEAM-8285</a> - Upgrade ZetaSQL to 2019.09.1.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8392">BEAM-8392</a> - Upgrade pyarrow version bounds: 0.15.1&lt;= to &lt;0.16.0.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-5895">BEAM-5895</a> - Upgrade com.rabbitmq:amqp-client to 5.7.3.</li>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-6896">BEAM-6896</a> - Upgrade PyYAML version bounds: 3.12&lt;= to &lt;6.0.0.</li>
-</ul>
-<h3 id="bugfixes">Bugfixes</h3>
-<ul>
-<li>[BEAM-8819] - AvroCoder for SpecificRecords is not serialized correctly since 2.13.0</li>
-<li>Various bug fixes and performance improvements.</li>
-</ul>
-<h3 id="known-issues">Known Issues</h3>
-<ul>
-<li><a href="https://issues.apache.org/jira/browse/BEAM-8989">BEAM-8989</a> Apache Nemo
-runner broken due to backwards incompatible change since 2.16.0.</li>
-</ul>
-<h2 id="list-of-contributors">List of Contributors</h2>
-<p>According to git shortlog, the following people contributed to the 2.17.0 release. Thank you to all contributors!</p>
-<p>Ahmet Altay, Alan Myrvold, Alexey Romanenko, Andre-Philippe Paquet, Andrew
-Pilloud, angulartist, Ankit Jhalaria, Ankur Goenka, Anton Kedin, Aryan Naraghi,
-Aurélien Geron, B M VISHWAS, Bartok Jozsef, Boyuan Zhang, Brian Hulette, Cerny
-Ondrej, Chad Dombrova, Chamikara Jayalath, ChethanU, cmach, Colm O hEigeartaigh,
-Cyrus Maden, Daniel Oliveira, Daniel Robert, Dante, David Cavazos, David
-Moravek, David Yan, Enrico Canzonieri, Etienne Chauchot, gxercavins, Hai Lu,
-Hannah Jiang, Ian Lance Taylor, Ismaël Mejía, Israel Herraiz, James Wen, Jan
-Lukavský, Jean-Baptiste Onofré, Jeff Klukas, jesusrv1103, Jofre, Kai Jiang,
-Kamil Wasilewski, Kasia Kucharczyk, Kenneth Knowles, Kirill Kozlov,
-kirillkozlov, Kohki YAMAGIWA, Kyle Weaver, Leonardo Alves Miguel, lloigor,
-lostluck, Luis Enrique Ortíz Ramirez, Luke Cwik, Mark Liu, Maximilian Michels,
-Michal Walenia, Mikhail Gryzykhin, mrociorg, Nicolas Delsaux, Ning Kang, NING
-KANG, Pablo Estrada, pabloem, Piotr Szczepanik, rahul8383, Rakesh Kumar, Renat
-Nasyrov, Reuven Lax, Robert Bradshaw, Robert Burke, Rui Wang, Ruslan Altynnikov,
-Ryan Skraba, Salman Raza, Saul Chavez, Sebastian Jambor, sunjincheng121, Tatu
-Saloranta, tchiarato, Thomas Weise, Tomo Suzuki, Tudor Marian, tvalentyn, Udi
-Meiri, Valentyn Tymofieiev, Viola Lyu, Vishwas, Yichi Zhang, Yifan Zou, Yueyang
-Qiu, Łukasz Gajowy</p></description><link>/blog/beam-2.17.0/</link><pubDate>Mon, 06 Jan 2020 00:00:01 -0800</pubDate><guid>/blog/beam-2.17.0/</guid><category>blog</category></item></channel></rss>
\ No newline at end of file
+<p>Ahmet Altay, Alex Amato, Alexey Romanenko, Andrew Pilloud, Ankur Goenka, Anton Kedin, Boyuan Zhang, Brian Hulette, Brian Martin, Chamikara Jayalath, Charles Chen, Craig Chambers, Daniel Oliveira, David Moravek, David Rieber, Dustin Rhodes, Etienne Chauchot, Gleb Kanterov, Hai Lu, Heejong Lee, Ismaël Mejía, Jan Lukavský, Jason Kuster, Jean-Baptiste Onofré, Jeff Klukas, João Cabrita, J Ross Thomson, Juan Rael, Juta, Kasia Kucharczyk, Kengo Seki, Kenneth Jung, Kenneth Knowles, Kyle We [...]
\ No newline at end of file
diff --git a/website/generated-content/get-started/index.xml b/website/generated-content/get-started/index.xml
index 4145620..cc25b73 100644
--- a/website/generated-content/get-started/index.xml
+++ b/website/generated-content/get-started/index.xml
@@ -1299,8 +1299,7 @@ barrenly: 1
...</code></pre>
</div>
<div class=runner-samza-local>
-<pre><code>
-$ more /tmp/counts*
+<pre><code>$ more /tmp/counts*
api: 7
are: 2
can: 2
diff --git a/website/generated-content/get-started/quickstart-java/index.html b/website/generated-content/get-started/quickstart-java/index.html
index 5d91268..31139b0 100644
--- a/website/generated-content/get-started/quickstart-java/index.html
+++ b/website/generated-content/get-started/quickstart-java/index.html
@@ -28,24 +28,24 @@ MinimalWordCount.java WordCount.java</code></pre></div><div class=shell-PowerShe
PS> dir
-...
-
-Mode LastWriteTime Length Name
----- ------------- ------ ----
-d----- 7/19/2018 11:00 PM src
+...
+
+Mode LastWriteTime Length Name
+---- ------------- ------ ----
+d----- 7/19/2018 11:00 PM src
-a---- 7/19/2018 11:00 PM 16051 pom.xml
PS> dir .\src\main\java\org\apache\beam\examples
...
-Mode LastWriteTime Length Name
----- ------------- ------ ----
-d----- 7/19/2018 11:00 PM common
-d----- 7/19/2018 11:00 PM complete
-d----- 7/19/2018 11:00 PM subprocess
--a---- 7/19/2018 11:00 PM 7073 DebuggingWordCount.java
--a---- 7/19/2018 11:00 PM 5945 MinimalWordCount.java
--a---- 7/19/2018 11:00 PM 9490 WindowedWordCount.java
+Mode LastWriteTime Length Name
+---- ------------- ------ ----
+d----- 7/19/2018 11:00 PM common
+d----- 7/19/2018 11:00 PM complete
+d----- 7/19/2018 11:00 PM subprocess
+-a---- 7/19/2018 11:00 PM 7073 DebuggingWordCount.java
+-a---- 7/19/2018 11:00 PM 5945 MinimalWordCount.java
+-a---- 7/19/2018 11:00 PM 9490 WindowedWordCount.java
-a---- 7/19/2018 11:00 PM 7662 WordCount.java</code></pre></div><p>For a detailed introduction to the Beam concepts used in these examples, see the <a href=/get-started/wordcount-example>WordCount Example Walkthrough</a>. Here, we’ll just focus on executing <code>WordCount.java</code>.</p><h2 id=optional-convert-from-maven-to-gradle-project>Optional: Convert from Maven to Gradle Project</h2><p>Ensure you are in the same directory as the <code>pom.xml</code> file g [...]
mavenCentral()
maven {
@@ -143,8 +143,7 @@ Bashful: 1
Below: 2
deserves: 32
barrenly: 1
-...</code></pre></div><div class=runner-samza-local><pre><code>
-$ more /tmp/counts*
+...</code></pre></div><div class=runner-samza-local><pre><code>$ more /tmp/counts*
api: 7
are: 2
can: 2
diff --git a/website/generated-content/get-started/wordcount-example/index.html b/website/generated-content/get-started/wordcount-example/index.html
index acee5f17..42aa01d 100644
--- a/website/generated-content/get-started/wordcount-example/index.html
+++ b/website/generated-content/get-started/wordcount-example/index.html
@@ -518,7 +518,7 @@ timestamp sometime in a 2-hour period.</p><div class=language-java><div class=hi
<span class=n>c</span><span class=o>.</span><span class=na>outputWithTimestamp</span><span class=o>(</span><span class=n>c</span><span class=o>.</span><span class=na>element</span><span class=o>(),</span> <span class=k>new</span> <span class=n>Instant</span><span class=o>(</span><span class=n>randomTimestamp</span><span class=o>));</span>
<span class=o>}</span>
<span class=o>}</span></code></pre></div></div><div class=language-py><div class=highlight><pre class=chroma><code class=language-py data-lang=py><span class=k>class</span> <span class=nc>AddTimestampFn</span><span class=p>(</span><span class=n>beam</span><span class=o>.</span><span class=n>DoFn</span><span class=p>):</span>
-
+
<span class=k>def</span> <span class=fm>__init__</span><span class=p>(</span><span class=bp>self</span><span class=p>,</span> <span class=n>min_timestamp</span><span class=p>,</span> <span class=n>max_timestamp</span><span class=p>):</span>
<span class=bp>self</span><span class=o>.</span><span class=n>min_timestamp</span> <span class=o>=</span> <span class=n>min_timestamp</span>
<span class=bp>self</span><span class=o>.</span><span class=n>max_timestamp</span> <span class=o>=</span> <span class=n>max_timestamp</span>
diff --git a/website/generated-content/index.html b/website/generated-content/index.html
index afeca34..642da30 100644
--- a/website/generated-content/index.html
+++ b/website/generated-content/index.html
@@ -5,7 +5,7 @@
<a class="button button--primary" href=/get-started/try-apache-beam/>Try Beam</a>
<a class="button button--primary" href=/get-started/downloads/>Download Beam SDK 2.23.0</a></div><div class=hero__ctas><a class=button href=/get-started/quickstart-java/>Java Quickstart</a>
<a class=button href=/get-started/quickstart-py/>Python Quickstart</a>
-<a class=button href=/get-started/quickstart-go/>Go Quickstart</a></div></div></div><div class=hero__cols__col><div class=hero__blog><div class=hero__blog__title>The latest from the blog</div><div class=hero__blog__cards><a class=hero__blog__cards__card href=/blog/beam-2.23.0/><div class=hero__blog__cards__card__title>Apache Beam 2.23.0</div><div class=hero__blog__cards__card__date>Jul 29, 2020</div></a><a class=hero__blog__cards__card href=/blog/beam-2.22.0/><div class=hero__blog__cards [...]
+<a class=button href=/get-started/quickstart-go/>Go Quickstart</a></div></div></div><div class=hero__cols__col><div class=hero__blog><div class=hero__blog__title>The latest from the blog</div><div class=hero__blog__cards><a class=hero__blog__cards__card href=/blog/python-improved-annotations/><div class=hero__blog__cards__card__title>Improved Annotation Support for the Python SDK</div><div class=hero__blog__cards__card__date>Aug 21, 2020</div></a><a class=hero__blog__cards__card href=/bl [...]
<a class="button button--primary" href=/get-started/downloads/>Download Beam SDK 2.23.0</a></div><div class=ctas__ctas><a class=button href=/get-started/quickstart-java/>Java Quickstart</a>
<a class=button href=/get-started/quickstart-py/>Python Quickstart</a>
<a class=button href=/get-started/quickstart-go/>Go Quickstart</a></div></div></div><footer class=footer><div class=footer__contained><div class=footer__cols><div class=footer__cols__col><div class=footer__cols__col__logo><img src=/images/beam_logo_circle.svg class=footer__logo alt="Beam logo"></div><div class=footer__cols__col__logo><img src=/images/apache_logo_circle.svg class=footer__logo alt="Apache logo"></div></div><div class="footer__cols__col footer__cols__col--md"><div class=foo [...]
diff --git a/website/generated-content/security/index.html b/website/generated-content/security/index.html
index 161c442..45326e9 100644
--- a/website/generated-content/security/index.html
+++ b/website/generated-content/security/index.html
@@ -6,11 +6,16 @@ Team</a> for reporting vulnerabilities. Note
that vulnerabilities should not be publicly disclosed until the project has
responded.</p><p>To report a possible security vulnerability, please email
<code>security@apache.org</code> and <code>pmc@beam.apache.org</code>. This is a non-public list
-that will reach the Beam PMC.</p><h1 id=known-security-issues>Known Security Issues</h1><h2 id=cve-2020-1929>CVE-2020-1929</h2><p>[CVE-2020-1929] Apache Beam MongoDB IO connector disables certificate trust verification</p><p>Severity: Major<br>Vendor: The Apache Software Foundation</p><p>Versions Affected:<br>Apache Beam 2.10.0 to 2.16.0</p><p>Description:<br>The Apache Beam MongoDB connector in versions 2.10.0 to 2.16.0 has an option to
+that will reach the Beam PMC.</p><h1 id=known-security-issues>Known Security Issues</h1><h2 id=cve-2020-1929>CVE-2020-1929</h2><p>[CVE-2020-1929] Apache Beam MongoDB IO connector disables certificate trust verification</p><p>Severity: Major
+Vendor: The Apache Software Foundation</p><p>Versions Affected:
+Apache Beam 2.10.0 to 2.16.0</p><p>Description:
+The Apache Beam MongoDB connector in versions 2.10.0 to 2.16.0 has an option to
disable SSL trust verification. However this configuration is not respected and
the certificate verification disables trust verification in every case. This
exclusion also gets registered globally which disables trust checking for any
-code running in the same JVM.</p><p>Mitigation:<br>Users of the affected versions should apply one of the following mitigations:</p><ul><li>Upgrade to Apache Beam 2.17.0 or later</li></ul><p>Acknowledgements:<br>This issue was reported (and fixed) by Colm Ó hÉigeartaigh.</p></div></div><footer class=footer><div class=footer__contained><div class=footer__cols><div class=footer__cols__col><div class=footer__cols__col__logo><img src=/images/beam_logo_circle.svg class=footer__logo alt="Beam [...]
+code running in the same JVM.</p><p>Mitigation:
+Users of the affected versions should apply one of the following mitigations:</p><ul><li>Upgrade to Apache Beam 2.17.0 or later</li></ul><p>Acknowledgements:
+This issue was reported (and fixed) by Colm Ó hÉigeartaigh.</p></div></div><footer class=footer><div class=footer__contained><div class=footer__cols><div class=footer__cols__col><div class=footer__cols__col__logo><img src=/images/beam_logo_circle.svg class=footer__logo alt="Beam logo"></div><div class=footer__cols__col__logo><img src=/images/apache_logo_circle.svg class=footer__logo alt="Apache logo"></div></div><div class="footer__cols__col footer__cols__col--md"><div class=footer__cols [...]
<a href=http://www.apache.org>The Apache Software Foundation</a>
| <a href=/privacy_policy>Privacy Policy</a>
| <a href=/feed.xml>RSS Feed</a><br><br>Apache Beam, Apache, Beam, the Beam logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation. All other products or name brands are trademarks of their respective holders, including The Apache Software Foundation.</div></footer></body></html>
\ No newline at end of file
diff --git a/website/generated-content/sitemap.xml b/website/generated-content/sitemap.xml
index 6ee9967..b9817de 100644
--- a/website/generated-content/sitemap.xml
+++ b/website/generated-content/sitemap.xml
@@ -1 +1 @@
-<?xml version="1.0" encoding="utf-8" standalone="yes"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml"><url><loc>/blog/beam-2.23.0/</loc><lastmod>2020-07-29T14:52:28-07:00</lastmod></url><url><loc>/categories/blog/</loc><lastmod>2020-07-29T14:52:28-07:00</lastmod></url><url><loc>/blog/</loc><lastmod>2020-07-29T14:52:28-07:00</lastmod></url><url><loc>/categories/</loc><lastmod>2020-07-29T14:52:28-07:00</lastmod></url><url><loc>/blog/b [...]
\ No newline at end of file
+<?xml version="1.0" encoding="utf-8" standalone="yes"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml"><url><loc>/categories/blog/</loc><lastmod>2020-08-26T13:09:05-05:00</lastmod></url><url><loc>/blog/</loc><lastmod>2020-08-26T13:09:05-05:00</lastmod></url><url><loc>/categories/</loc><lastmod>2020-08-26T13:09:05-05:00</lastmod></url><url><loc>/blog/python-improved-annotations/</loc><lastmod>2020-08-26T13:09:05-05:00</lastmod></url>< [...]
\ No newline at end of file