You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2019/06/07 18:13:59 UTC
[beam] branch asf-site updated: Publishing website 2019/06/07
18:13:48 at commit 39d695f
This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 0967859 Publishing website 2019/06/07 18:13:48 at commit 39d695f
0967859 is described below
commit 0967859d364837d5f9fe4259dfe86ea7c484ef9f
Author: jenkins <bu...@apache.org>
AuthorDate: Fri Jun 7 18:13:48 2019 +0000
Publishing website 2019/06/07 18:13:48 at commit 39d695f
---
.../documentation/execution-model/index.html | 20 +-
website/generated-content/documentation/index.html | 20 +-
.../io/built-in/google-bigquery/index.html | 20 +-
.../documentation/io/built-in/hadoop/index.html | 20 +-
.../documentation/io/built-in/hcatalog/index.html | 20 +-
.../documentation/io/built-in/index.html | 20 +-
.../documentation/io/developing-io-java/index.html | 20 +-
.../io/developing-io-overview/index.html | 20 +-
.../io/developing-io-python/index.html | 20 +-
.../documentation/io/testing/index.html | 20 +-
.../custom-io-patterns}/index.html | 71 +++---
.../file-processing-patterns}/index.html | 185 ++++++++-------
.../partition => patterns/overview}/index.html | 83 ++++---
.../pipeline-option-patterns}/index.html | 250 ++++++++++-----------
.../side-input-patterns}/index.html | 177 ++++++++-------
.../pipelines/create-your-pipeline/index.html | 20 +-
.../pipelines/design-your-pipeline/index.html | 20 +-
.../pipelines/test-your-pipeline/index.html | 20 +-
.../documentation/programming-guide/index.html | 20 +-
.../resources/learning-resources/index.html | 20 +-
.../resources/videos-and-podcasts/index.html | 20 +-
.../documentation/runners/jstorm/index.html | 20 +-
.../aggregation/approximatequantiles/index.html | 20 +-
.../aggregation/approximateunique/index.html | 20 +-
.../python/aggregation/cogroupbykey/index.html | 20 +-
.../python/aggregation/combineglobally/index.html | 20 +-
.../aggregation/combinewithcontext/index.html | 20 +-
.../transforms/python/aggregation/count/index.html | 20 +-
.../python/aggregation/distinct/index.html | 20 +-
.../python/aggregation/groupbykey/index.html | 20 +-
.../python/aggregation/groupintobatches/index.html | 20 +-
.../python/aggregation/latest/index.html | 20 +-
.../transforms/python/aggregation/max/index.html | 20 +-
.../transforms/python/aggregation/mean/index.html | 20 +-
.../transforms/python/aggregation/min/index.html | 20 +-
.../python/aggregation/sample/index.html | 20 +-
.../transforms/python/aggregation/sum/index.html | 20 +-
.../transforms/python/aggregation/top/index.html | 20 +-
.../python/elementwise/filter/index.html | 20 +-
.../python/elementwise/flatmap/index.html | 20 +-
.../transforms/python/elementwise/keys/index.html | 20 +-
.../python/elementwise/kvswap/index.html | 20 +-
.../transforms/python/elementwise/map/index.html | 20 +-
.../transforms/python/elementwise/pardo/index.html | 20 +-
.../python/elementwise/partition/index.html | 20 +-
.../transforms/python/elementwise/regex/index.html | 20 +-
.../transforms/python/elementwise/reify/index.html | 20 +-
.../python/elementwise/tostring/index.html | 20 +-
.../python/elementwise/values/index.html | 20 +-
.../python/elementwise/withkeys/index.html | 20 +-
.../python/elementwise/withtimestamps/index.html | 20 +-
.../transforms/python/other/create/index.html | 20 +-
.../transforms/python/other/flatten/index.html | 20 +-
.../transforms/python/other/passert/index.html | 20 +-
.../transforms/python/other/reshuffle/index.html | 20 +-
.../transforms/python/other/view/index.html | 20 +-
.../transforms/python/other/windowinto/index.html | 20 +-
.../transforms/python/overview/index.html | 20 +-
website/generated-content/js/section-nav.js | 6 +-
59 files changed, 1263 insertions(+), 569 deletions(-)
diff --git a/website/generated-content/documentation/execution-model/index.html b/website/generated-content/documentation/execution-model/index.html
index 587de2a..abc88ac 100644
--- a/website/generated-content/documentation/execution-model/index.html
+++ b/website/generated-content/documentation/execution-model/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/index.html b/website/generated-content/documentation/index.html
index 54f60ae..ea40f3e 100644
--- a/website/generated-content/documentation/index.html
+++ b/website/generated-content/documentation/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/io/built-in/google-bigquery/index.html b/website/generated-content/documentation/io/built-in/google-bigquery/index.html
index 3cef307..6b70e63 100644
--- a/website/generated-content/documentation/io/built-in/google-bigquery/index.html
+++ b/website/generated-content/documentation/io/built-in/google-bigquery/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/io/built-in/hadoop/index.html b/website/generated-content/documentation/io/built-in/hadoop/index.html
index ee96f01..023e445 100644
--- a/website/generated-content/documentation/io/built-in/hadoop/index.html
+++ b/website/generated-content/documentation/io/built-in/hadoop/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/io/built-in/hcatalog/index.html b/website/generated-content/documentation/io/built-in/hcatalog/index.html
index b9c7ab9..2d8ee21 100644
--- a/website/generated-content/documentation/io/built-in/hcatalog/index.html
+++ b/website/generated-content/documentation/io/built-in/hcatalog/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/io/built-in/index.html b/website/generated-content/documentation/io/built-in/index.html
index bc1f640..a484689 100644
--- a/website/generated-content/documentation/io/built-in/index.html
+++ b/website/generated-content/documentation/io/built-in/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/io/developing-io-java/index.html b/website/generated-content/documentation/io/developing-io-java/index.html
index 13a0baa..8c946d8 100644
--- a/website/generated-content/documentation/io/developing-io-java/index.html
+++ b/website/generated-content/documentation/io/developing-io-java/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/io/developing-io-overview/index.html b/website/generated-content/documentation/io/developing-io-overview/index.html
index 078d552..89652e8 100644
--- a/website/generated-content/documentation/io/developing-io-overview/index.html
+++ b/website/generated-content/documentation/io/developing-io-overview/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/io/developing-io-python/index.html b/website/generated-content/documentation/io/developing-io-python/index.html
index 48f95c5..e9247d6 100644
--- a/website/generated-content/documentation/io/developing-io-python/index.html
+++ b/website/generated-content/documentation/io/developing-io-python/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/io/testing/index.html b/website/generated-content/documentation/io/testing/index.html
index 93111f3..3573837 100644
--- a/website/generated-content/documentation/io/testing/index.html
+++ b/website/generated-content/documentation/io/testing/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html b/website/generated-content/documentation/patterns/custom-io-patterns/index.html
similarity index 88%
copy from website/generated-content/documentation/transforms/python/elementwise/partition/index.html
copy to website/generated-content/documentation/patterns/custom-io-patterns/index.html
index df7380b..d046839 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
+++ b/website/generated-content/documentation/patterns/custom-io-patterns/index.html
@@ -28,7 +28,7 @@
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Partition</title>
+ <title>Custom I/O patterns</title>
<meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow [...]
">
<link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400" rel="stylesheet">
@@ -39,7 +39,7 @@
<script src="/js/fix-menu.js"></script>
<script src="/js/section-nav.js"></script>
<script src="/js/page-nav.js"></script>
- <link rel="canonical" href="https://beam.apache.org/documentation/transforms/python/elementwise/partition/" data-proofer-ignore>
+ <link rel="canonical" href="https://beam.apache.org/documentation/patterns/custom-io-patterns/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.1/css/all.css" integrity="sha384-5sAR7xN1Nv6T6+dT2mhtzEpVJvfS3NScPQTrOxhwjIuvcA67KV2R5Jz6kr4abQsz" crossorigin="anonymous">
@@ -125,7 +125,7 @@
GitHub links will not resolve until the markdown source is available on the master branch.
New pages would fail validation during development / PR test automation.
-->
- <a href="https://github.com/apache/beam/edit/master/website/src/documentation/transforms/python/element-wise/partition.md" data-proofer-ignore>
+ <a href="https://github.com/apache/beam/edit/master/website/src/documentation/patterns/custom-io-patterns.md" data-proofer-ignore>
<i class="far fa-edit fa-lg" alt="Edit on GitHub" title="Edit on GitHub"></i>
</a>
</li>
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
@@ -347,8 +359,7 @@
<ul class="nav">
- <li><a href="#examples">Examples</a></li>
- <li><a href="#related-transforms">Related transforms</a></li>
+ <li><a href="#choosing-between-built-in-and-custom-connectors">Choosing between built-in and custom connectors</a></li>
</ul>
@@ -369,37 +380,31 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-<h1 id="partition">Partition</h1>
-<table align="left">
- <a target="_blank" class="button" href="https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.core.html#apache_beam.transforms.core.Partition">
- <img src="https://beam.apache.org/images/logos/sdks/python.png" width="20px" height="20px" alt="Pydoc" />
- Pydoc
- </a>
-</table>
-<p><br />
-Separates elements in a collection into multiple output
-collections. The partitioning function contains the logic that determines how
-to separate the elements of the input collection into each resulting
-partition output collection.</p>
+<h1 id="custom-io-patterns">Custom I/O patterns</h1>
-<p>The number of partitions must be determined at graph construction time.
-You cannot determine the number of partitions in mid-pipeline</p>
+<p>This page describes common patterns in pipelines with <a href="/documentation/io/developing-io-overview/">custom I/O connectors</a>. Custom I/O connectors connect pipelines to databases that aren’t supported by Beam’s <a href="/documentation/io/built-in/">built-in I/O transforms</a>.</p>
-<p>See more information in the <a href="/documentation/programming-guide/#partition">Beam Programming Guide</a>.</p>
+<nav class="language-switcher">
+ <strong>Adapt for:</strong>
+ <ul>
+ <li data-type="language-java" class="active">Java SDK</li>
+ <li data-type="language-py">Python SDK</li>
+ </ul>
+</nav>
+
+<h2 id="choosing-between-built-in-and-custom-connectors">Choosing between built-in and custom connectors</h2>
-<h2 id="examples">Examples</h2>
-<p>See <a href="https://issues.apache.org/jira/browse/BEAM-7389">BEAM-7389</a> for updates.</p>
+<p><a href="/documentation/io/built-in/">Built-in I/O connectors</a> are tested and hardened, so use them whenever possible. Only use custom I/O connectors when:</p>
-<h2 id="related-transforms">Related transforms</h2>
<ul>
- <li><a href="/documentation/transforms/python/elementwise/filter">Filter</a> is useful if the function is just
-deciding whether to output an element or not.</li>
- <li><a href="/documentation/transforms/python/elementwise/pardo">ParDo</a> is the most general element-wise mapping
-operation, and includes other abilities such as multiple output collections and side-inputs.</li>
- <li><a href="/documentation/transforms/python/aggregation/cogroupbykey">CoGroupByKey</a>
-performs a per-key equijoin.</li>
+ <li>No built-in options exist</li>
+ <li>Your pipeline pulls in a small subset of source data</li>
</ul>
+<p>For instance, use a custom I/O connector to enrich pipeline elements with a small subset of source data. If you’re processing a sales order and adding information to each purchase, you can use a custom I/O connector to pull the small subset of data into your pipeline (instead of processing the entire source).</p>
+
+<p>Beam distributes work across many threads, so custom I/O connectors can increase your data source’s load average. You can reduce the load with the <span class="language-java"><a href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/transforms/DoFn.StartBundle.html">start</a></span><span class="language-py"><a href="https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.core.html?highlight=bundle#apache_beam.transforms.core.DoFn.start_bundle">start</ [...]
+
</div>
</div>
<!--
diff --git a/website/generated-content/documentation/pipelines/create-your-pipeline/index.html b/website/generated-content/documentation/patterns/file-processing-patterns/index.html
similarity index 64%
copy from website/generated-content/documentation/pipelines/create-your-pipeline/index.html
copy to website/generated-content/documentation/patterns/file-processing-patterns/index.html
index 64e33ab..2b4997b 100644
--- a/website/generated-content/documentation/pipelines/create-your-pipeline/index.html
+++ b/website/generated-content/documentation/patterns/file-processing-patterns/index.html
@@ -28,7 +28,7 @@
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Create Your Pipeline</title>
+ <title>File processing patterns</title>
<meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow [...]
">
<link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400" rel="stylesheet">
@@ -39,7 +39,7 @@
<script src="/js/fix-menu.js"></script>
<script src="/js/section-nav.js"></script>
<script src="/js/page-nav.js"></script>
- <link rel="canonical" href="https://beam.apache.org/documentation/pipelines/create-your-pipeline/" data-proofer-ignore>
+ <link rel="canonical" href="https://beam.apache.org/documentation/patterns/file-processing-patterns/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.1/css/all.css" integrity="sha384-5sAR7xN1Nv6T6+dT2mhtzEpVJvfS3NScPQTrOxhwjIuvcA67KV2R5Jz6kr4abQsz" crossorigin="anonymous">
@@ -125,7 +125,7 @@
GitHub links will not resolve until the markdown source is available on the master branch.
New pages would fail validation during development / PR test automation.
-->
- <a href="https://github.com/apache/beam/edit/master/website/src/documentation/pipelines/create-your-pipeline.md" data-proofer-ignore>
+ <a href="https://github.com/apache/beam/edit/master/website/src/documentation/patterns/file-processing-patterns.md" data-proofer-ignore>
<i class="far fa-edit fa-lg" alt="Edit on GitHub" title="Edit on GitHub"></i>
</a>
</li>
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
@@ -347,12 +359,14 @@
<ul class="nav">
- <li><a href="#creating-your-pipeline-object">Creating Your Pipeline Object</a></li>
- <li><a href="#reading-data-into-your-pipeline">Reading Data Into Your Pipeline</a></li>
- <li><a href="#applying-transforms-to-process-pipeline-data">Applying Transforms to Process Pipeline Data</a></li>
- <li><a href="#writing-or-outputting-your-final-pipeline-data">Writing or Outputting Your Final Pipeline Data</a></li>
- <li><a href="#running-your-pipeline">Running Your Pipeline</a></li>
- <li><a href="#whats-next">What’s next</a></li>
+ <li><a href="#processing-files-as-they-arrive">Processing files as they arrive</a>
+ <ul>
+ <li><a href="#continuous-read-mode">Continuous read mode</a></li>
+ <li><a href="#stream-processing-triggered-from-external-source">Stream processing triggered from external source</a></li>
+ <li><a href="#batch-processing-triggered-from-external-source">Batch processing triggered from external source</a></li>
+ </ul>
+ </li>
+ <li><a href="#accessing-filenames">Accessing filenames</a></li>
</ul>
@@ -372,102 +386,123 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
-<h1 id="create-your-pipeline">Create Your Pipeline</h1>
-
-<ul id="markdown-toc">
- <li><a href="#creating-your-pipeline-object" id="markdown-toc-creating-your-pipeline-object">Creating Your Pipeline Object</a></li>
- <li><a href="#reading-data-into-your-pipeline" id="markdown-toc-reading-data-into-your-pipeline">Reading Data Into Your Pipeline</a></li>
- <li><a href="#applying-transforms-to-process-pipeline-data" id="markdown-toc-applying-transforms-to-process-pipeline-data">Applying Transforms to Process Pipeline Data</a></li>
- <li><a href="#writing-or-outputting-your-final-pipeline-data" id="markdown-toc-writing-or-outputting-your-final-pipeline-data">Writing or Outputting Your Final Pipeline Data</a></li>
- <li><a href="#running-your-pipeline" id="markdown-toc-running-your-pipeline">Running Your Pipeline</a></li>
- <li><a href="#whats-next" id="markdown-toc-whats-next">What’s next</a></li>
-</ul>
-<p>Your Beam program expresses a data processing pipeline, from start to finish. This section explains the mechanics of using the classes in the Beam SDKs to build a pipeline. To construct a pipeline using the classes in the Beam SDKs, your program will need to perform the following general steps:</p>
+<h1 id="file-processing-patterns">File processing patterns</h1>
-<ul>
- <li>Create a <code class="highlighter-rouge">Pipeline</code> object.</li>
- <li>Use a <strong>Read</strong> or <strong>Create</strong> transform to create one or more <code class="highlighter-rouge">PCollection</code>s for your pipeline data.</li>
- <li>Apply <strong>transforms</strong> to each <code class="highlighter-rouge">PCollection</code>. Transforms can change, filter, group, analyze, or otherwise process the elements in a <code class="highlighter-rouge">PCollection</code>. Each transform creates a new output <code class="highlighter-rouge">PCollection</code>, to which you can apply additional transforms until processing is complete.</li>
- <li><strong>Write</strong> or otherwise output the final, transformed <code class="highlighter-rouge">PCollection</code>s.</li>
- <li><strong>Run</strong> the pipeline.</li>
-</ul>
+<p>This page describes common file processing tasks. For more information on file-based I/O, see <a href="/documentation/programming-guide/#pipeline-io">Pipeline I/O</a> and <a href="/documentation/programming-guide/#file-based-data">File-based input and output data</a>.</p>
-<h2 id="creating-your-pipeline-object">Creating Your Pipeline Object</h2>
+<nav class="language-switcher">
+ <strong>Adapt for:</strong>
+ <ul>
+ <li data-type="language-java" class="active">Java SDK</li>
+ <li data-type="language-py">Python SDK</li>
+ </ul>
+</nav>
-<p>A Beam program often starts by creating a <code class="highlighter-rouge">Pipeline</code> object.</p>
+<h2 id="processing-files-as-they-arrive">Processing files as they arrive</h2>
-<p>In the Beam SDKs, each pipeline is represented by an explicit object of type <code class="highlighter-rouge">Pipeline</code>. Each <code class="highlighter-rouge">Pipeline</code> object is an independent entity that encapsulates both the data the pipeline operates over and the transforms that get applied to that data.</p>
+<p>This section shows you how to process files as they arrive in your file system or object store (like Google Cloud Storage). You can continuously read files or trigger stream and processing pipelines when a file arrives.</p>
-<p>To create a pipeline, declare a <code class="highlighter-rouge">Pipeline</code> object, and pass it some <a href="/documentation/programming-guide#configuring-pipeline-options">configuration options</a>.</p>
+<h3 id="continuous-read-mode">Continuous read mode</h3>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="c1">// Start by defining the options for the pipeline.</span>
-<span class="n">PipelineOptions</span> <span class="n">options</span> <span class="o">=</span> <span class="n">PipelineOptionsFactory</span><span class="o">.</span><span class="na">create</span><span class="o">();</span>
+<p class="language-java">You can use <code class="highlighter-rouge">FileIO</code> or <code class="highlighter-rouge">TextIO</code> to continuously read the source for new files.</p>
-<span class="c1">// Then create the pipeline.</span>
-<span class="n">Pipeline</span> <span class="n">p</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="o">.</span><span class="na">create</span><span class="o">(</span><span class="n">options</span><span class="o">);</span>
-</code></pre>
-</div>
+<p class="language-java">Use the <a href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/FileIO.html"><code class="highlighter-rouge">FileIO</code></a> class to continuously watch a single file pattern. The following example matches a file pattern repeatedly every 30 seconds, continuously returns new matched files as an unbounded <code class="highlighter-rouge">PCollection<Metadata></code>, and stops if no new files appear for one hour:</p>
-<h2 id="reading-data-into-your-pipeline">Reading Data Into Your Pipeline</h2>
+<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="c1">// This produces PCollection<MatchResult.Metadata></span>
+<span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">FileIO</span><span class="o">.</span><span class="na">match</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">filepattern</span><span class="o">(</span><span class="s">"..."</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">continuously</span><span class="o">(</span>
+ <span class="n">Duration</span><span class="o">.</span><span class="na">standardSeconds</span><span class="o">(</span><span class="mi">30</span><span class="o">),</span>
+ <span class="n">Watch</span><span class="o">.</span><span class="na">Growth</span><span class="o">.</span><span class="na">afterTimeSinceNewOutput</span><span class="o">(</span><span class="n">Duration</span><span class="o">.</span><span class="na">standardHours</span><span class="o">(</span><span class="mi">1</span><span class="o">))));</span>
-<p>To create your pipeline’s initial <code class="highlighter-rouge">PCollection</code>, you apply a root transform to your pipeline object. A root transform creates a <code class="highlighter-rouge">PCollection</code> from either an external data source or some local data you specify.</p>
+</code></pre>
+</div>
-<p>There are two kinds of root transforms in the Beam SDKs: <code class="highlighter-rouge">Read</code> and <code class="highlighter-rouge">Create</code>. <code class="highlighter-rouge">Read</code> transforms read data from an external source, such as a text file or a database table. <code class="highlighter-rouge">Create</code> transforms create a <code class="highlighter-rouge">PCollection</code> from an in-memory <code class="highlighter-rouge">java.util.Collection</code>.</p>
+<p class="language-java">The <a href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/TextIO.html"><code class="highlighter-rouge">TextIO</code></a> class <code class="highlighter-rouge">watchForNewFiles</code> property streams new file matches.</p>
-<p>The following example code shows how to <code class="highlighter-rouge">apply</code> a <code class="highlighter-rouge">TextIO.Read</code> root transform to read data from a text file. The transform is applied to a <code class="highlighter-rouge">Pipeline</code> object <code class="highlighter-rouge">p</code>, and returns a pipeline data set in the form of a <code class="highlighter-rouge">PCollection<String></code>:</p>
+<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="c1">// This produces PCollection<String></span>
+<span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">TextIO</span><span class="o">.</span><span class="na">read</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">from</span><span class="o">(</span><span class="s">"<path-to-files>/*"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">watchForNewFiles</span><span class="o">(</span>
+ <span class="c1">// Check for new files every minute</span>
+ <span class="n">Duration</span><span class="o">.</span><span class="na">standardMinutes</span><span class="o">(</span><span class="mi">1</span><span class="o">),</span>
+ <span class="c1">// Stop watching the filepattern if no new files appear within an hour</span>
+ <span class="n">Watch</span><span class="o">.</span><span class="na">Growth</span><span class="o">.</span><span class="na">afterTimeSinceNewOutput</span><span class="o">(</span><span class="n">Duration</span><span class="o">.</span><span class="na">standardHours</span><span class="o">(</span><span class="mi">1</span><span class="o">))));</span>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">lines</span> <span class="o">=</span> <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span>
- <span class="s">"ReadLines"</span><span class="o">,</span> <span class="n">TextIO</span><span class="o">.</span><span class="na">read</span><span class="o">().</span><span class="na">from</span><span class="o">(</span><span class="s">"gs://some/inputData.txt"</span><span class="o">));</span>
</code></pre>
</div>
-<h2 id="applying-transforms-to-process-pipeline-data">Applying Transforms to Process Pipeline Data</h2>
+<p class="language-java">Some runners may retain file lists during updates, but file lists don’t persist when you restart a pipeline. You can save file lists by:</p>
-<p>You can manipulate your data using the various <a href="/documentation/programming-guide/#transforms">transforms</a> provided in the Beam SDKs. To do this, you <strong>apply</strong> the trannsforms to your pipeline’s <code class="highlighter-rouge">PCollection</code> by calling the <code class="highlighter-rouge">apply</code> method on each <code class="highlighter-rouge">PCollection</code> that you want to process and passing the desired transform object as an argument.</p>
+<ul class="language-java">
+ <li>Storing processed filenames in an external file and deduplicating the lists at the next transform</li>
+ <li>Adding timestamps to filenames, writing a glob pattern to pull in only new files, and matching the pattern when the pipeline restarts</li>
+</ul>
-<p>The following code shows how to <code class="highlighter-rouge">apply</code> a transform to a <code class="highlighter-rouge">PCollection</code> of strings. The transform is a user-defined custom transform that reverses the contents of each string and outputs a new <code class="highlighter-rouge">PCollection</code> containing the reversed strings.</p>
+<p class="language-py">The continuous-read option is not available for Python.</p>
-<p>The input is a <code class="highlighter-rouge">PCollection<String></code> called <code class="highlighter-rouge">words</code>; the code passes an instance of a <code class="highlighter-rouge">PTransform</code> object called <code class="highlighter-rouge">ReverseWords</code> to <code class="highlighter-rouge">apply</code>, and saves the return value as the <code class="highlighter-rouge">PCollection<String></code> called <code class="highlighter-rouge">reversedWords</code>.</p>
+<h3 id="stream-processing-triggered-from-external-source">Stream processing triggered from external source</h3>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">words</span> <span class="o">=</span> <span class="o">...;</span>
+<p>A streaming pipeline can process data from an unbounded source. For example, to trigger stream processing with Google Cloud Pub/Sub:</p>
-<span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">reversedWords</span> <span class="o">=</span> <span class="n">words</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="k">new</span> <span class="n">ReverseWords</span><span class="o">());</span>
-</code></pre>
-</div>
+<ol>
+ <li>Use an external process to detect when new files arrive.</li>
+ <li>Send a Google Cloud Pub/Sub message with a URI to the file.</li>
+ <li>Access the URI from a <code class="highlighter-rouge">DoFn</code> that follows the Google Cloud Pub/Sub source.</li>
+ <li>Process the file.</li>
+</ol>
-<h2 id="writing-or-outputting-your-final-pipeline-data">Writing or Outputting Your Final Pipeline Data</h2>
+<h3 id="batch-processing-triggered-from-external-source">Batch processing triggered from external source</h3>
-<p>Once your pipeline has applied all of its transforms, you’ll usually need to output the results. To output your pipeline’s final <code class="highlighter-rouge">PCollection</code>s, you apply a <code class="highlighter-rouge">Write</code> transform to that <code class="highlighter-rouge">PCollection</code>. <code class="highlighter-rouge">Write</code> transforms can output the elements of a <code class="highlighter-rouge">PCollection</code> to an external data sink, such as a database [...]
+<p>To start or schedule a batch pipeline job when a file arrives, write the triggering event in the source file itself. This has the most latency because the pipeline must initialize before processing. It’s best suited for low-frequency, large, file-size updates.</p>
-<p>The following example code shows how to <code class="highlighter-rouge">apply</code> a <code class="highlighter-rouge">TextIO.Write</code> transform to write a <code class="highlighter-rouge">PCollection</code> of <code class="highlighter-rouge">String</code> to a text file:</p>
+<h2 id="accessing-filenames">Accessing filenames</h2>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">filteredWords</span> <span class="o">=</span> <span class="o">...;</span>
+<p class="language-java">Use the <code class="highlighter-rouge">FileIO</code> class to read filenames in a pipeline job. <code class="highlighter-rouge">FileIO</code> returns a <code class="highlighter-rouge">PCollection<ReadableFile></code> object, and the <code class="highlighter-rouge">ReadableFile</code> instance contains the filename.</p>
-<span class="n">filteredWords</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="s">"WriteMyFile"</span><span class="o">,</span> <span class="n">TextIO</span><span class="o">.</span><span class="na">write</span><span class="o">().</span><span class="na">to</span><span class="o">(</span><span class="s">"gs://some/outputData.txt"</span><span class="o">));</span>
-</code></pre>
-</div>
+<p class="language-java">To access filenames:</p>
-<h2 id="running-your-pipeline">Running Your Pipeline</h2>
+<ol class="language-java">
+ <li>Create a <code class="highlighter-rouge">ReadableFile</code> instance with <code class="highlighter-rouge">FileIO</code>. <code class="highlighter-rouge">FileIO</code> returns a <code class="highlighter-rouge">PCollection<ReadableFile></code> object. The <code class="highlighter-rouge">ReadableFile</code> class contains the filename.</li>
+ <li>Call the <code class="highlighter-rouge">readFullyAsUTF9String()</code> method to read the file into memory and return the filename as a <code class="highlighter-rouge">String</code> object. If memory is limited, you can use utility classes like <a href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/FileSystems.html"><code class="highlighter-rouge">FileSystems</code></a> to work directly with the file.</li>
+</ol>
-<p>Once you have constructed your pipeline, use the <code class="highlighter-rouge">run</code> method to execute the pipeline. Pipelines are executed asynchronously: the program you create sends a specification for your pipeline to a <strong>pipeline runner</strong>, which then constructs and runs the actual series of pipeline operations.</p>
+<p class="language-py">To read filenames in a pipeline job:</p>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">p</span><span class="o">.</span><span class="na">run</span><span class="o">();</span>
-</code></pre>
-</div>
+<ol class="language-py">
+ <li>Collect the list of file URIs. You can use the <a href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.filesystems.html?highlight=filesystems#module-apache_beam.io.filesystems"><code class="highlighter-rouge">FileSystems</code></a> module to get a list of files that match a glob pattern.</li>
+ <li>Pass the file URIs to a <code class="highlighter-rouge">PCollection</code>.</li>
+</ol>
-<p>The <code class="highlighter-rouge">run</code> method is asynchronous. If you’d like a blocking execution instead, run your pipeline appending the <code class="highlighter-rouge">waitUntilFinish</code> method:</p>
+<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">FileIO</span><span class="o">.</span><span class="na">match</span><span class="o">().</span><span class="na">filepattern</span><span class="o">(</span><span class="s">"hdfs://path/to/*.gz"</span><span class="o">))</span>
+ <span class="c1">// withCompression can be omitted - by default compression is detected from the filename.</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">FileIO</span><span class="o">.</span><span class="na">readMatches</span><span class="o">().</span><span class="na">withCompression</span><span class="o">(</span><span class="n">Compression</span><span class="o">.</span><span class="na">GZIP</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">ParDo</span><span class="o">.</span><span class="na">of</span><span class="o">(</span>
+ <span class="k">new</span> <span class="n">DoFn</span><span class="o"><</span><span class="n">FileIO</span><span class="o">.</span><span class="na">ReadableFile</span><span class="o">,</span> <span class="n">String</span><span class="o">>()</span> <span class="o">{</span>
+ <span class="nd">@ProcessElement</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="nf">process</span><span class="o">(</span><span class="nd">@Element</span> <span class="n">FileIO</span><span class="o">.</span><span class="na">ReadableFile</span> <span class="n">file</span><span class="o">)</span> <span class="o">{</span>
+ <span class="c1">// We now have access to the file and its metadata</span>
+ <span class="n">LOG</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">"File Metadata resourceId is {} "</span><span class="o">,</span> <span class="n">file</span><span class="o">.</span><span class="na">getMetadata</span><span class="o">().</span><span class="na">resourceId</span><span class="o">());</span>
+ <span class="o">}</span>
+ <span class="o">}));</span>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">p</span><span class="o">.</span><span class="na">run</span><span class="o">().</span><span class="na">waitUntilFinish</span><span class="o">();</span>
</code></pre>
</div>
+<div class="language-py highlighter-rouge"><pre class="highlight"><code><span class="k">with</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">()</span> <span class="k">as</span> <span class="n">p</span><span class="p">:</span>
+ <span class="n">readable_files</span> <span class="o">=</span> <span class="p">(</span><span class="n">p</span>
+ <span class="o">|</span> <span class="n">fileio</span><span class="o">.</span><span class="n">MatchFiles</span><span class="p">(</span><span class="s">'hdfs://path/to/*.txt'</span><span class="p">)</span>
+ <span class="o">|</span> <span class="n">fileio</span><span class="o">.</span><span class="n">ReadMatches</span><span class="p">()</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Reshuffle</span><span class="p">())</span>
+ <span class="n">files_and_contents</span> <span class="o">=</span> <span class="p">(</span><span class="n">readable_files</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">path</span><span class="p">,</span>
+ <span class="n">x</span><span class="o">.</span><span class="n">read_utf8</span><span class="p">())))</span>
-<h2 id="whats-next">What’s next</h2>
-
-<ul>
- <li><a href="/documentation/programming-guide">Programming Guide</a> - Learn the details of creating your pipeline, configuring pipeline options, and applying transforms.</li>
- <li><a href="/documentation/pipelines/test-your-pipeline">Test your pipeline</a>.</li>
-</ul>
+</code></pre>
+</div>
</div>
</div>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html b/website/generated-content/documentation/patterns/overview/index.html
similarity index 88%
copy from website/generated-content/documentation/transforms/python/elementwise/partition/index.html
copy to website/generated-content/documentation/patterns/overview/index.html
index df7380b..291256c 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
+++ b/website/generated-content/documentation/patterns/overview/index.html
@@ -28,7 +28,7 @@
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Partition</title>
+ <title>Overview</title>
<meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow [...]
">
<link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400" rel="stylesheet">
@@ -39,7 +39,7 @@
<script src="/js/fix-menu.js"></script>
<script src="/js/section-nav.js"></script>
<script src="/js/page-nav.js"></script>
- <link rel="canonical" href="https://beam.apache.org/documentation/transforms/python/elementwise/partition/" data-proofer-ignore>
+ <link rel="canonical" href="https://beam.apache.org/documentation/patterns/overview/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.1/css/all.css" integrity="sha384-5sAR7xN1Nv6T6+dT2mhtzEpVJvfS3NScPQTrOxhwjIuvcA67KV2R5Jz6kr4abQsz" crossorigin="anonymous">
@@ -125,7 +125,7 @@
GitHub links will not resolve until the markdown source is available on the master branch.
New pages would fail validation during development / PR test automation.
-->
- <a href="https://github.com/apache/beam/edit/master/website/src/documentation/transforms/python/element-wise/partition.md" data-proofer-ignore>
+ <a href="https://github.com/apache/beam/edit/master/website/src/documentation/patterns/overview.md" data-proofer-ignore>
<i class="far fa-edit fa-lg" alt="Edit on GitHub" title="Edit on GitHub"></i>
</a>
</li>
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
@@ -347,8 +359,8 @@
<ul class="nav">
- <li><a href="#examples">Examples</a></li>
- <li><a href="#related-transforms">Related transforms</a></li>
+ <li><a href="#contributing-a-pattern">Contributing a pattern</a></li>
+ <li><a href="#whats-next">What’s next</a></li>
</ul>
@@ -369,35 +381,40 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-<h1 id="partition">Partition</h1>
-<table align="left">
- <a target="_blank" class="button" href="https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.core.html#apache_beam.transforms.core.Partition">
- <img src="https://beam.apache.org/images/logos/sdks/python.png" width="20px" height="20px" alt="Pydoc" />
- Pydoc
- </a>
-</table>
-<p><br />
-Separates elements in a collection into multiple output
-collections. The partitioning function contains the logic that determines how
-to separate the elements of the input collection into each resulting
-partition output collection.</p>
+<h1 id="common-pipeline-patterns">Common pipeline patterns</h1>
+
+<p>Pipeline patterns demonstrate common Beam use cases. Pipeline patterns are based on real-world Beam deployments. Each pattern has a description, examples, and a solution or psuedocode.</p>
+
+<p><strong>File processing patterns</strong> - Patterns for reading from and writing to files</p>
+<ul>
+ <li><a href="/documentation/patterns/file-processing-patterns/#processing-files-as-they-arrive">Processing files as they arrive</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/#accessing-filenames">Accessing filenames</a></li>
+</ul>
+
+<p><strong>Side input patterns</strong> - Patterns for processing supplementary data</p>
+<ul>
+ <li><a href="/documentation/patterns/side-input-patterns/#using-global-window-side-inputs-in-non-global-windows">Using global window side inputs in non-global windows</a></li>
+</ul>
+
+<p><strong>Pipeline option patterns</strong> - Patterns for configuring pipelines</p>
+<ul>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/#retroactively-logging-runtime-parameters">Retroactively logging runtime parameters</a></li>
+</ul>
+
+<p><strong>Custom I/O patterns</strong></p>
+<ul>
+ <li><a href="/documentation/patterns/custom-io-patterns/#choosing-between-built-in-and-custom-connectors">Choosing between built-in and custom connectors</a></li>
+</ul>
-<p>The number of partitions must be determined at graph construction time.
-You cannot determine the number of partitions in mid-pipeline</p>
+<h2 id="contributing-a-pattern">Contributing a pattern</h2>
-<p>See more information in the <a href="/documentation/programming-guide/#partition">Beam Programming Guide</a>.</p>
+<p>To contribute a new pipeline pattern, create an issue with the <a href="https://issues.apache.org/jira/browse/BEAM-7449?jql=labels%20%3D%20pipeline-patterns"><code class="highlighter-rouge">pipeline-patterns</code> label</a> and add details to the issue description. See <a href="/contribute/">Get started contributing</a> for more information.</p>
-<h2 id="examples">Examples</h2>
-<p>See <a href="https://issues.apache.org/jira/browse/BEAM-7389">BEAM-7389</a> for updates.</p>
+<h2 id="whats-next">What’s next</h2>
-<h2 id="related-transforms">Related transforms</h2>
<ul>
- <li><a href="/documentation/transforms/python/elementwise/filter">Filter</a> is useful if the function is just
-deciding whether to output an element or not.</li>
- <li><a href="/documentation/transforms/python/elementwise/pardo">ParDo</a> is the most general element-wise mapping
-operation, and includes other abilities such as multiple output collections and side-inputs.</li>
- <li><a href="/documentation/transforms/python/aggregation/cogroupbykey">CoGroupByKey</a>
-performs a per-key equijoin.</li>
+ <li>Try an <a href="/get-started/try-apache-beam/">end-to-end example</a></li>
+ <li>Execute your pipeline on a <a href="/documentation/runners/capability-matrix/">runner</a></li>
</ul>
</div>
diff --git a/website/generated-content/documentation/io/built-in/hcatalog/index.html b/website/generated-content/documentation/patterns/pipeline-option-patterns/index.html
similarity index 63%
copy from website/generated-content/documentation/io/built-in/hcatalog/index.html
copy to website/generated-content/documentation/patterns/pipeline-option-patterns/index.html
index b9c7ab9..88ecb67 100644
--- a/website/generated-content/documentation/io/built-in/hcatalog/index.html
+++ b/website/generated-content/documentation/patterns/pipeline-option-patterns/index.html
@@ -28,7 +28,7 @@
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Apache HCatalog InputFormat IO</title>
+ <title>Pipeline option patterns</title>
<meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow [...]
">
<link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400" rel="stylesheet">
@@ -39,7 +39,7 @@
<script src="/js/fix-menu.js"></script>
<script src="/js/section-nav.js"></script>
<script src="/js/page-nav.js"></script>
- <link rel="canonical" href="https://beam.apache.org/documentation/io/built-in/hcatalog/" data-proofer-ignore>
+ <link rel="canonical" href="https://beam.apache.org/documentation/patterns/pipeline-option-patterns/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.1/css/all.css" integrity="sha384-5sAR7xN1Nv6T6+dT2mhtzEpVJvfS3NScPQTrOxhwjIuvcA67KV2R5Jz6kr4abQsz" crossorigin="anonymous">
@@ -125,7 +125,7 @@
GitHub links will not resolve until the markdown source is available on the master branch.
New pages would fail validation during development / PR test automation.
-->
- <a href="https://github.com/apache/beam/edit/master/website/src/documentation/io/built-in-hcatalog.md" data-proofer-ignore>
+ <a href="https://github.com/apache/beam/edit/master/website/src/documentation/patterns/pipeline-option-patterns.md" data-proofer-ignore>
<i class="far fa-edit fa-lg" alt="Edit on GitHub" title="Edit on GitHub"></i>
</a>
</li>
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
@@ -347,9 +359,7 @@
<ul class="nav">
- <li><a href="#reading-using-hcatalogio">Reading using HCatalogIO</a></li>
- <li><a href="#writing-using-hcatalogio">Writing using HCatalogIO</a></li>
- <li><a href="#using-older-versions-of-hcatalog-1x">Using older versions of HCatalog (1.x)</a></li>
+ <li><a href="#retroactively-logging-runtime-parameters">Retroactively logging runtime parameters</a></li>
</ul>
@@ -370,145 +380,117 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-<h1 id="hcatalog-io">HCatalog IO</h1>
+<h1 id="pipeline-option-patterns">Pipeline option patterns</h1>
-<p>An <code class="highlighter-rouge">HCatalogIO</code> is a transform for reading and writing data to an HCatalog managed source.</p>
+<p>The samples on this page show you common pipeline configurations. For more information about pipeline configuration options, see <a href="/documentation/programming-guide/#creating-a-pipeline">Creating a pipeline</a> and <a href="/documentation/programming-guide/#configuring-pipeline-options">Configuring pipeline options</a>.</p>
-<h3 id="reading-using-hcatalogio">Reading using HCatalogIO</h3>
+<nav class="language-switcher">
+ <strong>Adapt for:</strong>
+ <ul>
+ <li data-type="language-java" class="active">Java SDK</li>
+ <li data-type="language-py">Python SDK</li>
+ </ul>
+</nav>
-<p>To configure an HCatalog source, you must specify a metastore URI and a table name. Other optional parameters are database and filter.</p>
+<h2 id="retroactively-logging-runtime-parameters">Retroactively logging runtime parameters</h2>
-<p>For example:</p>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">></span> <span class="n">configProperties</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HashMap</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">>() [...]
-<span class="n">configProperties</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"hive.metastore.uris"</span><span class="o">,</span><span class="s">"thrift://metastore-host:port"</span><span class="o">);</span>
-<span class="n">pipeline</span>
- <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">HCatalogIO</span><span class="o">.</span><span class="na">read</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withConfigProperties</span><span class="o">(</span><span class="n">configProperties</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withDatabase</span><span class="o">(</span><span class="s">"default"</span><span class="o">)</span> <span class="c1">//optional, assumes default if none specified</span>
- <span class="o">.</span><span class="na">withTable</span><span class="o">(</span><span class="s">"employee"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withFilter</span><span class="o">(</span><span class="n">filterString</span><span class="o">)</span> <span class="c1">//optional, may be specified if the table is partitioned</span>
-</code></pre>
-</div>
-<div class="language-py highlighter-rouge"><pre class="highlight"><code> <span class="c"># The Beam SDK for Python does not support HCatalogIO.</span>
-</code></pre>
-</div>
+<p>Use the <code class="highlighter-rouge">ValueProvider</code> interface to access runtime parameters after completing a pipeline job.</p>
-<h3 id="writing-using-hcatalogio">Writing using HCatalogIO</h3>
+<p>You can use the <code class="highlighter-rouge">ValueProvider</code> interface to pass runtime parameters to your pipeline, but you can only log the parameters from within the the Beam DAG. A solution is to add a pipeline <a href="/documentation/programming-guide/#applying-transforms">branch</a> with a <code class="highlighter-rouge">DoFn</code> that processes a placeholder value and then logs the runtime parameters:</p>
-<p>To configure an <code class="highlighter-rouge">HCatalog</code> sink, you must specify a metastore URI and a table name. Other
-optional parameters are database, partition and batchsize.
-The destination table should exist beforehand as the transform will not create a new table if missing.</p>
+<div class="language-java highlighter-rouge"><pre class="highlight"><code>
+<span class="cm">/** Sample of PipelineOptions with a ValueProvider option argument. */</span>
+<span class="kd">public</span> <span class="kd">interface</span> <span class="nc">MyOptions</span> <span class="kd">extends</span> <span class="n">PipelineOptions</span> <span class="o">{</span>
+ <span class="nd">@Description</span><span class="o">(</span><span class="s">"My option"</span><span class="o">)</span>
+ <span class="nd">@Default</span><span class="o">.</span><span class="na">String</span><span class="o">(</span><span class="s">"Hello world!"</span><span class="o">)</span>
+ <span class="n">ValueProvider</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="nf">getStringValue</span><span class="o">();</span>
-<p>For example:</p>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">></span> <span class="n">configProperties</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HashMap</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">>() [...]
-<span class="n">configProperties</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"hive.metastore.uris"</span><span class="o">,</span><span class="s">"thrift://metastore-host:port"</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="nf">setStringValue</span><span class="o">(</span><span class="n">ValueProvider</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">value</span><span class="o">);</span>
+<span class="o">}</span>
+
+<span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">accessingValueProviderInfoAfterRunSnip1</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="o">{</span>
+
+ <span class="n">MyOptions</span> <span class="n">options</span> <span class="o">=</span> <span class="n">PipelineOptionsFactory</span><span class="o">.</span><span class="na">fromArgs</span><span class="o">(</span><span class="n">args</span><span class="o">).</span><span class="na">withValidation</span><span class="o">().</span><span class="na">as</span><span class="o">(</span><span class="n">MyOptions</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
+
+ <span class="c1">// Create pipeline.</span>
+ <span class="n">Pipeline</span> <span class="n">p</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="o">.</span><span class="na">create</span><span class="o">(</span><span class="n">options</span><span class="o">);</span>
+
+ <span class="c1">// Add a branch for logging the ValueProvider value.</span>
+ <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Create</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="mi">1</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">ParDo</span><span class="o">.</span><span class="na">of</span><span class="o">(</span>
+ <span class="k">new</span> <span class="n">DoFn</span><span class="o"><</span><span class="n">Integer</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>()</span> <span class="o">{</span>
+
+ <span class="c1">// Define the DoFn that logs the ValueProvider value.</span>
+ <span class="nd">@ProcessElement</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="nf">process</span><span class="o">(</span><span class="n">ProcessContext</span> <span class="n">c</span><span class="o">)</span> <span class="o">{</span>
+
+ <span class="n">MyOptions</span> <span class="n">ops</span> <span class="o">=</span> <span class="n">c</span><span class="o">.</span><span class="na">getPipelineOptions</span><span class="o">().</span><span class="na">as</span><span class="o">(</span><span class="n">MyOptions</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
+ <span class="c1">// This example logs the ValueProvider value, but you could store it by</span>
+ <span class="c1">// pushing it to an external database.</span>
+
+ <span class="n">LOG</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">"Option StringValue was {}"</span><span class="o">,</span> <span class="n">ops</span><span class="o">.</span><span class="na">getStringValue</span><span class="o">());</span>
+ <span class="o">}</span>
+ <span class="o">}));</span>
+
+ <span class="c1">// The main pipeline.</span>
+ <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Create</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">2</span><span class="o">,</span> <span class="mi">3</span><span class="o">,</span> <span class="mi">4</span><span class="o">)).</span><span class="na">apply</span><span class="o">(</span><span class="n">Sum</span><span [...]
+
+ <span class="n">p</span><span class="o">.</span><span class="na">run</span><span class="o">();</span>
+<span class="o">}</span>
-<span class="n">pipeline</span>
- <span class="o">.</span><span class="na">apply</span><span class="o">(...)</span>
- <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">HCatalogIO</span><span class="o">.</span><span class="na">write</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withConfigProperties</span><span class="o">(</span><span class="n">configProperties</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withDatabase</span><span class="o">(</span><span class="s">"default"</span><span class="o">)</span> <span class="c1">//optional, assumes default if none specified</span>
- <span class="o">.</span><span class="na">withTable</span><span class="o">(</span><span class="s">"employee"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withPartition</span><span class="o">(</span><span class="n">partitionValues</span><span class="o">)</span> <span class="c1">//optional, may be specified if the table is partitioned</span>
- <span class="o">.</span><span class="na">withBatchSize</span><span class="o">(</span><span class="mi">1024L</span><span class="o">))</span> <span class="c1">//optional, assumes a default batch size of 1024 if none specified</span>
-</code></pre>
-</div>
-<div class="language-py highlighter-rouge"><pre class="highlight"><code> <span class="c"># The Beam SDK for Python does not support HCatalogIO.</span>
-</code></pre>
-</div>
-<h3 id="using-older-versions-of-hcatalog-1x">Using older versions of HCatalog (1.x)</h3>
-
-<p><code class="highlighter-rouge">HCatalogIO</code> is build for Apache HCatalog versions 2 and up and will not work out of the box for older versions of HCatalog.
-The following illustrates a workaround to work with Hive 1.1.</p>
-
-<p>Include the following Hive 1.2 jars in the über jar you build.
-The 1.2 jars provide the necessary methods for Beam while remain compatible with Hive 1.1.</p>
-
-<div class="highlighter-rouge"><pre class="highlight"><code><dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-io-hcatalog</artifactId>
- <version>${beam.version}</version>
-</dependency>
-<dependency>
- <groupId>org.apache.hive.hcatalog</groupId>
- <artifactId>hive-hcatalog-core</artifactId>
- <version>1.2</version>
-</dependency>
-<dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>1.2</version>
-</dependency>
-<dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>1.2</version>
-</dependency>
-<dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- <version>1.2</version>
-</dependency>
</code></pre>
</div>
-<p>Relocate <em>only</em> the following hive packages:</p>
-
-<div class="highlighter-rouge"><pre class="highlight"><code><plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven-shade-plugin.version}</version>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>shaded</shadedClassifierName>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- </transformers>
- <relocations>
- <!-- Important: Do not relocate org.apache.hadoop.hive -->
- <relocation>
- <pattern>org.apache.hadoop.hive.conf</pattern>
- <shadedPattern>h12.org.apache.hadoop.hive.conf</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hadoop.hive.ql</pattern>
- <shadedPattern>h12.org.apache.hadoop.hive.ql</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hadoop.hive.metastore</pattern>
- <shadedPattern>h12.org.apache.hadoop.hive.metastore</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
-</plugin>
+<div class="language-py highlighter-rouge"><pre class="highlight"><code><span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">import</span> <span class="nn">apache_beam</span> <span class="kn">as</span> <span class="nn">beam</span>
+<span class="kn">from</span> <span class="nn">apache_beam.options.pipeline_options</span> <span class="kn">import</span> <span class="n">PipelineOptions</span>
+<span class="kn">from</span> <span class="nn">apache_beam.utils.value_provider</span> <span class="kn">import</span> <span class="n">RuntimeValueProvider</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io</span> <span class="kn">import</span> <span class="n">WriteToText</span>
+
+<span class="k">class</span> <span class="nc">MyOptions</span><span class="p">(</span><span class="n">PipelineOptions</span><span class="p">):</span>
+ <span class="nd">@classmethod</span>
+ <span class="k">def</span> <span class="nf">_add_argparse_args</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">parser</span><span class="p">):</span>
+ <span class="n">parser</span><span class="o">.</span><span class="n">add_value_provider_argument</span><span class="p">(</span><span class="s">'--string_value'</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">LogValueProvidersFn</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">DoFn</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string_vp</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">string_vp</span> <span class="o">=</span> <span class="n">string_vp</span>
+
+ <span class="c"># Define the DoFn that logs the ValueProvider value.</span>
+ <span class="c"># The DoFn is called when creating the pipeline branch.</span>
+ <span class="c"># This example logs the ValueProvider value, but</span>
+ <span class="c"># you could store it by pushing it to an external database.</span>
+ <span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">an_int</span><span class="p">):</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">'The string_value is </span><span class="si">%</span><span class="s">s'</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">string_vp</span><span class="o">.</span><span class="n">get</span><span class="p">())</span>
+ <span class="c"># Another option (where you don't need to pass the value at all) is:</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s">'The string value is </span><span class="si">%</span><span class="s">s'</span> <span class="o">%</span>
+ <span class="n">RuntimeValueProvider</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s">'string_value'</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="s">''</span><span class="p">))</span>
+
+<span class="n">pipeline_options</span> <span class="o">=</span> <span class="n">PipelineOptions</span><span class="p">()</span>
+<span class="c"># Create pipeline.</span>
+<span class="n">p</span> <span class="o">=</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">(</span><span class="n">options</span><span class="o">=</span><span class="n">pipeline_options</span><span class="p">)</span>
+
+<span class="n">my_options</span> <span class="o">=</span> <span class="n">pipeline_options</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">MyOptions</span><span class="p">)</span>
+<span class="c"># Add a branch for logging the ValueProvider value.</span>
+<span class="n">_</span> <span class="o">=</span> <span class="p">(</span><span class="n">p</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="bp">None</span><span class="p">])</span>
+ <span class="o">|</span> <span class="s">'LogValueProvs'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">ParDo</span><span class="p">(</span>
+ <span class="n">LogValueProvidersFn</span><span class="p">(</span><span class="n">my_options</span><span class="o">.</span><span class="n">string_value</span><span class="p">)))</span>
+
+<span class="c"># The main pipeline.</span>
+<span class="n">result_pc</span> <span class="o">=</span> <span class="p">(</span><span class="n">p</span>
+ <span class="o">|</span> <span class="s">"main_pc"</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span>
+ <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">combiners</span><span class="o">.</span><span class="n">Sum</span><span class="o">.</span><span class="n">Globally</span><span class="p">())</span>
+
+<span class="n">p</span><span class="o">.</span><span class="n">run</span><span class="p">()</span><span class="o">.</span><span class="n">wait_until_finish</span><span class="p">()</span>
+
+
</code></pre>
</div>
-<p>This has been testing to read SequenceFile and ORCFile file backed tables running with
-Beam 2.4.0 on Spark 2.3 / YARN in a Cloudera CDH 5.12.2 managed environment.</p>
-
</div>
</div>
<!--
diff --git a/website/generated-content/documentation/pipelines/create-your-pipeline/index.html b/website/generated-content/documentation/patterns/side-input-patterns/index.html
similarity index 61%
copy from website/generated-content/documentation/pipelines/create-your-pipeline/index.html
copy to website/generated-content/documentation/patterns/side-input-patterns/index.html
index 64e33ab..36b0782 100644
--- a/website/generated-content/documentation/pipelines/create-your-pipeline/index.html
+++ b/website/generated-content/documentation/patterns/side-input-patterns/index.html
@@ -28,7 +28,7 @@
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Create Your Pipeline</title>
+ <title>Side input patterns</title>
<meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow [...]
">
<link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400" rel="stylesheet">
@@ -39,7 +39,7 @@
<script src="/js/fix-menu.js"></script>
<script src="/js/section-nav.js"></script>
<script src="/js/page-nav.js"></script>
- <link rel="canonical" href="https://beam.apache.org/documentation/pipelines/create-your-pipeline/" data-proofer-ignore>
+ <link rel="canonical" href="https://beam.apache.org/documentation/patterns/side-input-patterns/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.1/css/all.css" integrity="sha384-5sAR7xN1Nv6T6+dT2mhtzEpVJvfS3NScPQTrOxhwjIuvcA67KV2R5Jz6kr4abQsz" crossorigin="anonymous">
@@ -125,7 +125,7 @@
GitHub links will not resolve until the markdown source is available on the master branch.
New pages would fail validation during development / PR test automation.
-->
- <a href="https://github.com/apache/beam/edit/master/website/src/documentation/pipelines/create-your-pipeline.md" data-proofer-ignore>
+ <a href="https://github.com/apache/beam/edit/master/website/src/documentation/patterns/side-input-patterns.md" data-proofer-ignore>
<i class="far fa-edit fa-lg" alt="Edit on GitHub" title="Edit on GitHub"></i>
</a>
</li>
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
@@ -347,12 +359,7 @@
<ul class="nav">
- <li><a href="#creating-your-pipeline-object">Creating Your Pipeline Object</a></li>
- <li><a href="#reading-data-into-your-pipeline">Reading Data Into Your Pipeline</a></li>
- <li><a href="#applying-transforms-to-process-pipeline-data">Applying Transforms to Process Pipeline Data</a></li>
- <li><a href="#writing-or-outputting-your-final-pipeline-data">Writing or Outputting Your Final Pipeline Data</a></li>
- <li><a href="#running-your-pipeline">Running Your Pipeline</a></li>
- <li><a href="#whats-next">What’s next</a></li>
+ <li><a href="#using-global-window-side-inputs-in-non-global-windows">Using global window side inputs in non-global windows</a></li>
</ul>
@@ -372,103 +379,113 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
-<h1 id="create-your-pipeline">Create Your Pipeline</h1>
-
-<ul id="markdown-toc">
- <li><a href="#creating-your-pipeline-object" id="markdown-toc-creating-your-pipeline-object">Creating Your Pipeline Object</a></li>
- <li><a href="#reading-data-into-your-pipeline" id="markdown-toc-reading-data-into-your-pipeline">Reading Data Into Your Pipeline</a></li>
- <li><a href="#applying-transforms-to-process-pipeline-data" id="markdown-toc-applying-transforms-to-process-pipeline-data">Applying Transforms to Process Pipeline Data</a></li>
- <li><a href="#writing-or-outputting-your-final-pipeline-data" id="markdown-toc-writing-or-outputting-your-final-pipeline-data">Writing or Outputting Your Final Pipeline Data</a></li>
- <li><a href="#running-your-pipeline" id="markdown-toc-running-your-pipeline">Running Your Pipeline</a></li>
- <li><a href="#whats-next" id="markdown-toc-whats-next">What’s next</a></li>
-</ul>
-<p>Your Beam program expresses a data processing pipeline, from start to finish. This section explains the mechanics of using the classes in the Beam SDKs to build a pipeline. To construct a pipeline using the classes in the Beam SDKs, your program will need to perform the following general steps:</p>
+<h1 id="side-input-patterns">Side input patterns</h1>
-<ul>
- <li>Create a <code class="highlighter-rouge">Pipeline</code> object.</li>
- <li>Use a <strong>Read</strong> or <strong>Create</strong> transform to create one or more <code class="highlighter-rouge">PCollection</code>s for your pipeline data.</li>
- <li>Apply <strong>transforms</strong> to each <code class="highlighter-rouge">PCollection</code>. Transforms can change, filter, group, analyze, or otherwise process the elements in a <code class="highlighter-rouge">PCollection</code>. Each transform creates a new output <code class="highlighter-rouge">PCollection</code>, to which you can apply additional transforms until processing is complete.</li>
- <li><strong>Write</strong> or otherwise output the final, transformed <code class="highlighter-rouge">PCollection</code>s.</li>
- <li><strong>Run</strong> the pipeline.</li>
-</ul>
+<p>The samples on this page show you common Beam side input patterns. A side input is an additional input that your <code class="highlighter-rouge">DoFn</code> can access each time it processes an element in the input <code class="highlighter-rouge">PCollection</code>. For more information, see the <a href="/documentation/programming-guide/#side-inputs">programming guide section on side inputs</a>.</p>
-<h2 id="creating-your-pipeline-object">Creating Your Pipeline Object</h2>
+<h2 id="using-global-window-side-inputs-in-non-global-windows">Using global window side inputs in non-global windows</h2>
-<p>A Beam program often starts by creating a <code class="highlighter-rouge">Pipeline</code> object.</p>
+<p>You can retrieve side inputs from global windows to use them in a pipeline job with non-global windows, like a <code class="highlighter-rouge">FixedWindow</code>.</p>
-<p>In the Beam SDKs, each pipeline is represented by an explicit object of type <code class="highlighter-rouge">Pipeline</code>. Each <code class="highlighter-rouge">Pipeline</code> object is an independent entity that encapsulates both the data the pipeline operates over and the transforms that get applied to that data.</p>
+<p>To use global window side inputs in pipelines with non-global windows:</p>
-<p>To create a pipeline, declare a <code class="highlighter-rouge">Pipeline</code> object, and pass it some <a href="/documentation/programming-guide#configuring-pipeline-options">configuration options</a>.</p>
+<ol>
+ <li>
+ <p>Write a <code class="highlighter-rouge">DoFn</code> that periodically pulls data from a bounded source into a global window.</p>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="c1">// Start by defining the options for the pipeline.</span>
-<span class="n">PipelineOptions</span> <span class="n">options</span> <span class="o">=</span> <span class="n">PipelineOptionsFactory</span><span class="o">.</span><span class="na">create</span><span class="o">();</span>
+ <p>a. Use the <code class="highlighter-rouge">GenerateSequence</code> source transform to periodically emit a value.</p>
-<span class="c1">// Then create the pipeline.</span>
-<span class="n">Pipeline</span> <span class="n">p</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="o">.</span><span class="na">create</span><span class="o">(</span><span class="n">options</span><span class="o">);</span>
-</code></pre>
-</div>
+ <p>b. Instantiate a data-driven trigger that activates on each element and pulls data from a bounded source.</p>
-<h2 id="reading-data-into-your-pipeline">Reading Data Into Your Pipeline</h2>
+ <p>c. Fire the trigger to pass the data into the global window.</p>
+ </li>
+ <li>
+ <p>Create the side input for downstream transforms. The side input should fit into memory.</p>
+ </li>
+</ol>
-<p>To create your pipeline’s initial <code class="highlighter-rouge">PCollection</code>, you apply a root transform to your pipeline object. A root transform creates a <code class="highlighter-rouge">PCollection</code> from either an external data source or some local data you specify.</p>
+<p>The global window side input triggers on processing time, so the main pipeline nondeterministically matches the side input to elements in event time.</p>
-<p>There are two kinds of root transforms in the Beam SDKs: <code class="highlighter-rouge">Read</code> and <code class="highlighter-rouge">Create</code>. <code class="highlighter-rouge">Read</code> transforms read data from an external source, such as a text file or a database table. <code class="highlighter-rouge">Create</code> transforms create a <code class="highlighter-rouge">PCollection</code> from an in-memory <code class="highlighter-rouge">java.util.Collection</code>.</p>
+<p>For instance, the following code sample uses a <code class="highlighter-rouge">Map</code> to create a <code class="highlighter-rouge">DoFn</code>. The <code class="highlighter-rouge">Map</code> becomes a <code class="highlighter-rouge">View.asSingleton</code> side input that’s rebuilt on each counter tick. The side input updates every 5 seconds in order to demonstrate the workflow. In a real-world scenario, the side input would typically update every few hours or once per day.</p>
-<p>The following example code shows how to <code class="highlighter-rouge">apply</code> a <code class="highlighter-rouge">TextIO.Read</code> root transform to read data from a text file. The transform is applied to a <code class="highlighter-rouge">Pipeline</code> object <code class="highlighter-rouge">p</code>, and returns a pipeline data set in the form of a <code class="highlighter-rouge">PCollection<String></code>:</p>
+<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">sideInputPatterns</span><span class="o">()</span> <span class="o">{</span>
+ <span class="c1">// Using View.asSingleton, this pipeline uses a dummy external service as illustration.</span>
+ <span class="c1">// Run in debug mode to see the output</span>
+ <span class="n">Pipeline</span> <span class="n">p</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="o">.</span><span class="na">create</span><span class="o">();</span>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">lines</span> <span class="o">=</span> <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span>
- <span class="s">"ReadLines"</span><span class="o">,</span> <span class="n">TextIO</span><span class="o">.</span><span class="na">read</span><span class="o">().</span><span class="na">from</span><span class="o">(</span><span class="s">"gs://some/inputData.txt"</span><span class="o">));</span>
-</code></pre>
-</div>
+ <span class="c1">// Create slowly updating sideinput</span>
-<h2 id="applying-transforms-to-process-pipeline-data">Applying Transforms to Process Pipeline Data</h2>
+ <span class="n">PCollectionView</span><span class="o"><</span><span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">>></span> <span class="n">map</span> <span class="o">=</span>
+ <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">GenerateSequence</span><span class="o">.</span><span class="na">from</span><span class="o">(</span><span class="mi">0</span><span class="o">).</span><span class="na">withRate</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="n">Duration</span><span class="o">.</span><span class="na">standardSeconds</span><span class="o">(</s [...]
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">Window</span><span class="o">.<</span><span class="n">Long</span><span class="o">></span><span class="n">into</span><span class="o">(</span><span class="k">new</span> <span class="n">GlobalWindows</span><span class="o">())</span>
+ <span class="o">.</span><span class="na">triggering</span><span class="o">(</span><span class="n">Repeatedly</span><span class="o">.</span><span class="na">forever</span><span class="o">(</span><span class="n">AfterProcessingTime</span><span class="o">.</span><span class="na">pastFirstElementInPane</span><span class="o">()))</span>
+ <span class="o">.</span><span class="na">discardingFiredPanes</span><span class="o">())</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">ParDo</span><span class="o">.</span><span class="na">of</span><span class="o">(</span>
+ <span class="k">new</span> <span class="n">DoFn</span><span class="o"><</span><span class="n">Long</span><span class="o">,</span> <span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">>>()</span> <span class="o">{</span>
-<p>You can manipulate your data using the various <a href="/documentation/programming-guide/#transforms">transforms</a> provided in the Beam SDKs. To do this, you <strong>apply</strong> the trannsforms to your pipeline’s <code class="highlighter-rouge">PCollection</code> by calling the <code class="highlighter-rouge">apply</code> method on each <code class="highlighter-rouge">PCollection</code> that you want to process and passing the desired transform object as an argument.</p>
+ <span class="nd">@ProcessElement</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="nf">process</span><span class="o">(</span>
+ <span class="nd">@Element</span> <span class="n">Long</span> <span class="n">input</span><span class="o">,</span> <span class="n">OutputReceiver</span><span class="o"><</span><span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">>></span> <span class="n">o</span><span class="o">)</span> <span class="o">{</span>
+ <span class="c1">// Do any external reads needed here...</span>
+ <span class="c1">// We will make use of our dummy external service.</span>
+ <span class="c1">// Every time this triggers, the complete map will be replaced with that</span>
+ <span class="c1">// read from</span>
+ <span class="c1">// the service.</span>
+ <span class="n">o</span><span class="o">.</span><span class="na">output</span><span class="o">(</span><span class="n">DummyExternalService</span><span class="o">.</span><span class="na">readDummyData</span><span class="o">());</span>
+ <span class="o">}</span>
+ <span class="o">}))</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">View</span><span class="o">.</span><span class="na">asSingleton</span><span class="o">());</span>
-<p>The following code shows how to <code class="highlighter-rouge">apply</code> a transform to a <code class="highlighter-rouge">PCollection</code> of strings. The transform is a user-defined custom transform that reverses the contents of each string and outputs a new <code class="highlighter-rouge">PCollection</code> containing the reversed strings.</p>
+ <span class="c1">// ---- Consume slowly updating sideinput</span>
-<p>The input is a <code class="highlighter-rouge">PCollection<String></code> called <code class="highlighter-rouge">words</code>; the code passes an instance of a <code class="highlighter-rouge">PTransform</code> object called <code class="highlighter-rouge">ReverseWords</code> to <code class="highlighter-rouge">apply</code>, and saves the return value as the <code class="highlighter-rouge">PCollection<String></code> called <code class="highlighter-rouge">reversedWords</code>.</p>
+ <span class="c1">// GenerateSequence is only used here to generate dummy data for this illustration.</span>
+ <span class="c1">// You would use your real source for example PubSubIO, KafkaIO etc...</span>
+ <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">GenerateSequence</span><span class="o">.</span><span class="na">from</span><span class="o">(</span><span class="mi">0</span><span class="o">).</span><span class="na">withRate</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="n">Duration</span><span class="o">.</span><span class="na">standardSeconds</span><span class="o">(</span> [...]
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Window</span><span class="o">.</span><span class="na">into</span><span class="o">(</span><span class="n">FixedWindows</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">Duration</span><span class="o">.</span><span class="na">standardSeconds</span><span class="o">(</span><span class="mi">1</span><span class="o">))))</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Sum</span><span class="o">.</span><span class="na">longsGlobally</span><span class="o">().</span><span class="na">withoutDefaults</span><span class="o">())</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">ParDo</span><span class="o">.</span><span class="na">of</span><span class="o">(</span>
+ <span class="k">new</span> <span class="n">DoFn</span><span class="o"><</span><span class="n">Long</span><span class="o">,</span> <span class="n">KV</span><span class="o"><</span><span class="n">Long</span><span class="o">,</span> <span class="n">Long</span><span class="o">>>()</span> <span class="o">{</span>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">words</span> <span class="o">=</span> <span class="o">...;</span>
+ <span class="nd">@ProcessElement</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="nf">process</span><span class="o">(</span><span class="n">ProcessContext</span> <span class="n">c</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">></span> <span class="n">keyMap</span> <span class="o">=</span> <span class="n">c</span><span class="o">.</span><span class="na">sideInput</span><span class="o">(</span><span class="n">map</span><span class="o">);</span>
+ <span class="n">c</span><span class="o">.</span><span class="na">outputWithTimestamp</span><span class="o">(</span><span class="n">KV</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="mi">1L</span><span class="o">,</span> <span class="n">c</span><span class="o">.</span><span class="na">element</span><span class="o">()),</span> <span class="n">Instant</span><span class="o">.</span><span class="na">now</span><span class="o">( [...]
-<span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">reversedWords</span> <span class="o">=</span> <span class="n">words</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="k">new</span> <span class="n">ReverseWords</span><span class="o">());</span>
-</code></pre>
-</div>
+ <span class="n">LOG</span><span class="o">.</span><span class="na">debug</span><span class="o">(</span>
+ <span class="s">"Value is {} key A is {} and key B is {}"</span><span class="o">,</span>
+ <span class="n">c</span><span class="o">.</span><span class="na">element</span><span class="o">(),</span>
+ <span class="n">keyMap</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="s">"Key_A"</span><span class="o">),</span>
+ <span class="n">keyMap</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="s">"Key_B"</span><span class="o">));</span>
+ <span class="o">}</span>
+ <span class="o">})</span>
+ <span class="o">.</span><span class="na">withSideInputs</span><span class="o">(</span><span class="n">map</span><span class="o">));</span>
+<span class="o">}</span>
-<h2 id="writing-or-outputting-your-final-pipeline-data">Writing or Outputting Your Final Pipeline Data</h2>
+<span class="cm">/** Dummy class representing a pretend external service. */</span>
+<span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">DummyExternalService</span> <span class="o">{</span>
-<p>Once your pipeline has applied all of its transforms, you’ll usually need to output the results. To output your pipeline’s final <code class="highlighter-rouge">PCollection</code>s, you apply a <code class="highlighter-rouge">Write</code> transform to that <code class="highlighter-rouge">PCollection</code>. <code class="highlighter-rouge">Write</code> transforms can output the elements of a <code class="highlighter-rouge">PCollection</code> to an external data sink, such as a database [...]
+ <span class="kd">public</span> <span class="kd">static</span> <span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">></span> <span class="nf">readDummyData</span><span class="o">()</span> <span class="o">{</span>
-<p>The following example code shows how to <code class="highlighter-rouge">apply</code> a <code class="highlighter-rouge">TextIO.Write</code> transform to write a <code class="highlighter-rouge">PCollection</code> of <code class="highlighter-rouge">String</code> to a text file:</p>
+ <span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">></span> <span class="n">map</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HashMap</span><span class="o"><>();</span>
+ <span class="n">Instant</span> <span class="n">now</span> <span class="o">=</span> <span class="n">Instant</span><span class="o">.</span><span class="na">now</span><span class="o">();</span>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">filteredWords</span> <span class="o">=</span> <span class="o">...;</span>
+ <span class="n">DateTimeFormatter</span> <span class="n">dtf</span> <span class="o">=</span> <span class="n">DateTimeFormat</span><span class="o">.</span><span class="na">forPattern</span><span class="o">(</span><span class="s">"HH:MM:SS"</span><span class="o">);</span>
-<span class="n">filteredWords</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="s">"WriteMyFile"</span><span class="o">,</span> <span class="n">TextIO</span><span class="o">.</span><span class="na">write</span><span class="o">().</span><span class="na">to</span><span class="o">(</span><span class="s">"gs://some/outputData.txt"</span><span class="o">));</span>
-</code></pre>
-</div>
+ <span class="n">map</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"Key_A"</span><span class="o">,</span> <span class="n">now</span><span class="o">.</span><span class="na">minus</span><span class="o">(</span><span class="n">Duration</span><span class="o">.</span><span class="na">standardSeconds</span><span class="o">(</span><span class="mi">30</span><span class="o">)).</span><span class="na">toString</span><span class="o">(</span><sp [...]
+ <span class="n">map</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"Key_B"</span><span class="o">,</span> <span class="n">now</span><span class="o">.</span><span class="na">minus</span><span class="o">(</span><span class="n">Duration</span><span class="o">.</span><span class="na">standardSeconds</span><span class="o">(</span><span class="mi">30</span><span class="o">)).</span><span class="na">toString</span><span class="o">());</span>
-<h2 id="running-your-pipeline">Running Your Pipeline</h2>
+ <span class="k">return</span> <span class="n">map</span><span class="o">;</span>
+ <span class="o">}</span>
+<span class="o">}</span>
-<p>Once you have constructed your pipeline, use the <code class="highlighter-rouge">run</code> method to execute the pipeline. Pipelines are executed asynchronously: the program you create sends a specification for your pipeline to a <strong>pipeline runner</strong>, which then constructs and runs the actual series of pipeline operations.</p>
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">p</span><span class="o">.</span><span class="na">run</span><span class="o">();</span>
</code></pre>
</div>
-<p>The <code class="highlighter-rouge">run</code> method is asynchronous. If you’d like a blocking execution instead, run your pipeline appending the <code class="highlighter-rouge">waitUntilFinish</code> method:</p>
-
-<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">p</span><span class="o">.</span><span class="na">run</span><span class="o">().</span><span class="na">waitUntilFinish</span><span class="o">();</span>
-</code></pre>
-</div>
-
-<h2 id="whats-next">What’s next</h2>
-
-<ul>
- <li><a href="/documentation/programming-guide">Programming Guide</a> - Learn the details of creating your pipeline, configuring pipeline options, and applying transforms.</li>
- <li><a href="/documentation/pipelines/test-your-pipeline">Test your pipeline</a>.</li>
-</ul>
-
</div>
</div>
<!--
diff --git a/website/generated-content/documentation/pipelines/create-your-pipeline/index.html b/website/generated-content/documentation/pipelines/create-your-pipeline/index.html
index 64e33ab..69ad3e0 100644
--- a/website/generated-content/documentation/pipelines/create-your-pipeline/index.html
+++ b/website/generated-content/documentation/pipelines/create-your-pipeline/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/pipelines/design-your-pipeline/index.html b/website/generated-content/documentation/pipelines/design-your-pipeline/index.html
index b40c062..6cc3e79 100644
--- a/website/generated-content/documentation/pipelines/design-your-pipeline/index.html
+++ b/website/generated-content/documentation/pipelines/design-your-pipeline/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/pipelines/test-your-pipeline/index.html b/website/generated-content/documentation/pipelines/test-your-pipeline/index.html
index a8ecad7..c451478 100644
--- a/website/generated-content/documentation/pipelines/test-your-pipeline/index.html
+++ b/website/generated-content/documentation/pipelines/test-your-pipeline/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/programming-guide/index.html b/website/generated-content/documentation/programming-guide/index.html
index 55aeee1..e72fc92 100644
--- a/website/generated-content/documentation/programming-guide/index.html
+++ b/website/generated-content/documentation/programming-guide/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/resources/learning-resources/index.html b/website/generated-content/documentation/resources/learning-resources/index.html
index 54c5772..d6b1f6b 100644
--- a/website/generated-content/documentation/resources/learning-resources/index.html
+++ b/website/generated-content/documentation/resources/learning-resources/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/resources/videos-and-podcasts/index.html b/website/generated-content/documentation/resources/videos-and-podcasts/index.html
index 21a3228..674b6c5 100644
--- a/website/generated-content/documentation/resources/videos-and-podcasts/index.html
+++ b/website/generated-content/documentation/resources/videos-and-podcasts/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/runners/jstorm/index.html b/website/generated-content/documentation/runners/jstorm/index.html
index e999f3e..f3fcd45 100644
--- a/website/generated-content/documentation/runners/jstorm/index.html
+++ b/website/generated-content/documentation/runners/jstorm/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/approximatequantiles/index.html b/website/generated-content/documentation/transforms/python/aggregation/approximatequantiles/index.html
index bd94616..c8fedc6 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/approximatequantiles/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/approximatequantiles/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/approximateunique/index.html b/website/generated-content/documentation/transforms/python/aggregation/approximateunique/index.html
index 8ecc1e2..e9dec2b 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/approximateunique/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/approximateunique/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/cogroupbykey/index.html b/website/generated-content/documentation/transforms/python/aggregation/cogroupbykey/index.html
index fb15e23..cad2d58 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/cogroupbykey/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/cogroupbykey/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/combineglobally/index.html b/website/generated-content/documentation/transforms/python/aggregation/combineglobally/index.html
index 5adf05f..0542647 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/combineglobally/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/combineglobally/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/combinewithcontext/index.html b/website/generated-content/documentation/transforms/python/aggregation/combinewithcontext/index.html
index 374547c..3e52c89 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/combinewithcontext/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/combinewithcontext/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/count/index.html b/website/generated-content/documentation/transforms/python/aggregation/count/index.html
index 553219e..c4f9079 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/count/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/count/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/distinct/index.html b/website/generated-content/documentation/transforms/python/aggregation/distinct/index.html
index f0de107..82f5780 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/distinct/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/distinct/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/groupbykey/index.html b/website/generated-content/documentation/transforms/python/aggregation/groupbykey/index.html
index cb41768..4d1df83 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/groupbykey/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/groupbykey/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/groupintobatches/index.html b/website/generated-content/documentation/transforms/python/aggregation/groupintobatches/index.html
index e2a5e1b..1239ebe 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/groupintobatches/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/groupintobatches/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/latest/index.html b/website/generated-content/documentation/transforms/python/aggregation/latest/index.html
index b1cd04a..f2e1f02 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/latest/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/latest/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/max/index.html b/website/generated-content/documentation/transforms/python/aggregation/max/index.html
index 9ca7fdf..4269a1e 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/max/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/max/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/mean/index.html b/website/generated-content/documentation/transforms/python/aggregation/mean/index.html
index 4059a36..570412b 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/mean/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/mean/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/min/index.html b/website/generated-content/documentation/transforms/python/aggregation/min/index.html
index 1271578..4b83d96 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/min/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/min/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/sample/index.html b/website/generated-content/documentation/transforms/python/aggregation/sample/index.html
index eaba924..00d9416 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/sample/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/sample/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/sum/index.html b/website/generated-content/documentation/transforms/python/aggregation/sum/index.html
index acd0461..0bcbc59 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/sum/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/sum/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/aggregation/top/index.html b/website/generated-content/documentation/transforms/python/aggregation/top/index.html
index 782d6b9..619ed8a 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/top/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/top/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/filter/index.html b/website/generated-content/documentation/transforms/python/elementwise/filter/index.html
index 3280ac2..c3f9918 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/filter/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/filter/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html b/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html
index ad34689..4d6fb64 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/keys/index.html b/website/generated-content/documentation/transforms/python/elementwise/keys/index.html
index 6ac3e7a..c4b7706 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/keys/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/keys/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html b/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html
index 0a5cda0..885f3d1 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/map/index.html b/website/generated-content/documentation/transforms/python/elementwise/map/index.html
index 489818d..e2856a6 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/map/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/map/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html b/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html
index 350a87d..7e71e47 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html b/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
index df7380b..5c7f72d 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/regex/index.html b/website/generated-content/documentation/transforms/python/elementwise/regex/index.html
index 00e2125..d6d4a345 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/regex/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/regex/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/reify/index.html b/website/generated-content/documentation/transforms/python/elementwise/reify/index.html
index 37da776..aac27e7 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/reify/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/reify/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html b/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html
index 5cee866..f267e60 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/values/index.html b/website/generated-content/documentation/transforms/python/elementwise/values/index.html
index 7e92f25..9a90b24 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/values/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/values/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/withkeys/index.html b/website/generated-content/documentation/transforms/python/elementwise/withkeys/index.html
index f4c63fc..4b958c4 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/withkeys/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/withkeys/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/elementwise/withtimestamps/index.html b/website/generated-content/documentation/transforms/python/elementwise/withtimestamps/index.html
index e870448..6f3a51f 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/withtimestamps/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/withtimestamps/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/other/create/index.html b/website/generated-content/documentation/transforms/python/other/create/index.html
index c09b239..8373334 100644
--- a/website/generated-content/documentation/transforms/python/other/create/index.html
+++ b/website/generated-content/documentation/transforms/python/other/create/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/other/flatten/index.html b/website/generated-content/documentation/transforms/python/other/flatten/index.html
index 4088751..febd212 100644
--- a/website/generated-content/documentation/transforms/python/other/flatten/index.html
+++ b/website/generated-content/documentation/transforms/python/other/flatten/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/other/passert/index.html b/website/generated-content/documentation/transforms/python/other/passert/index.html
index 7bd1997..2165aaa 100644
--- a/website/generated-content/documentation/transforms/python/other/passert/index.html
+++ b/website/generated-content/documentation/transforms/python/other/passert/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/other/reshuffle/index.html b/website/generated-content/documentation/transforms/python/other/reshuffle/index.html
index c49be06..aa99829 100644
--- a/website/generated-content/documentation/transforms/python/other/reshuffle/index.html
+++ b/website/generated-content/documentation/transforms/python/other/reshuffle/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/other/view/index.html b/website/generated-content/documentation/transforms/python/other/view/index.html
index 6adb455..579f8b4 100644
--- a/website/generated-content/documentation/transforms/python/other/view/index.html
+++ b/website/generated-content/documentation/transforms/python/other/view/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/other/windowinto/index.html b/website/generated-content/documentation/transforms/python/other/windowinto/index.html
index a7ccfe0..69423da 100644
--- a/website/generated-content/documentation/transforms/python/other/windowinto/index.html
+++ b/website/generated-content/documentation/transforms/python/other/windowinto/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/documentation/transforms/python/overview/index.html b/website/generated-content/documentation/transforms/python/overview/index.html
index f2d583b..db6329c 100644
--- a/website/generated-content/documentation/transforms/python/overview/index.html
+++ b/website/generated-content/documentation/transforms/python/overview/index.html
@@ -289,11 +289,11 @@
<li><a href="/documentation/transforms/python/aggregation/cogroupbykey/">CoGroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/combineglobally/">CombineGlobally</a></li>
<li><a href="/documentation/transforms/python/aggregation/count/">Count</a></li>
- <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/distinct/">Distinct</a></li>
<li><a href="/documentation/transforms/python/aggregation/groupbykey/">GroupByKey</a></li>
<li><a href="/documentation/transforms/python/aggregation/mean/">Mean</a></li>
- <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
- <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/sample/">Sample</a></li>
+ <li><a href="/documentation/transforms/python/aggregation/top/">Top</a></li>
</ul>
</li>
<li class="section-nav-item--collapsible">
@@ -303,7 +303,7 @@
<li><a href="/documentation/transforms/python/other/create/">Create</a></li>
<li><a href="/documentation/transforms/python/other/flatten/">Flatten</a></li>
<li><a href="/documentation/transforms/python/other/reshuffle/">Reshuffle</a></li>
- <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
+ <li><a href="/documentation/transforms/python/other/windowinto/">WindowInto</a></li>
</ul>
</li>
</ul>
@@ -312,6 +312,18 @@
</li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Common pipeline patterns</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/patterns/overview/">Overview</a></li>
+ <li><a href="/documentation/patterns/file-processing-patterns/">File processing patterns</a></li>
+ <li><a href="/documentation/patterns/side-input-patterns/">Side input patterns</a></li>
+ <li><a href="/documentation/patterns/pipeline-option-patterns/">Pipeline option patterns</a></li>
+ <li><a href="/documentation/patterns/custom-io-patterns/">Custom I/O patterns</a></li>
+ </ul>
+</li>
+
+<li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Learning resources</span>
<ul class="section-nav-list">
diff --git a/website/generated-content/js/section-nav.js b/website/generated-content/js/section-nav.js
index 11e1c30..ad8e523 100644
--- a/website/generated-content/js/section-nav.js
+++ b/website/generated-content/js/section-nav.js
@@ -95,8 +95,10 @@ $(document).ready(function () {
var _self = this;
var sectionNavEl = $("." + idSectionNav);
var sectionNavHeight = $(sectionNavEl).height();
+ var mainContent = $(".container-main-content");
- $(".container-main-content").css({"min-height": sectionNavHeight});
+ mainContent.css({"min-height": sectionNavHeight});
+ sectionNavEl.css({"max-height": mainContent.css("height")});
$(window).resize(function () {
if ($(window).width() > CONST.DESKTOP_BREAKPOINT) {
@@ -166,4 +168,4 @@ $(document).ready(function () {
"classNameNavActiveItem": ".section-nav a.active"
}
).init();
-});
+});
\ No newline at end of file