You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/05/27 06:02:52 UTC
[beam] branch asf-site updated: Publishing website 2020/05/27
06:02:42 at commit 1a5dd77
This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 870749f Publishing website 2020/05/27 06:02:42 at commit 1a5dd77
870749f is described below
commit 870749f381fb494fcd9c417e7088e3c5297aefba
Author: jenkins <bu...@apache.org>
AuthorDate: Wed May 27 06:02:43 2020 +0000
Publishing website 2020/05/27 06:02:42 at commit 1a5dd77
---
website/generated-content/documentation/index.html | 13 +
website/generated-content/documentation/index.xml | 1788 ++++++++++++++++++--
.../io/built-in/google-bigquery/index.html | 13 +
.../documentation/io/built-in/hadoop/index.html | 13 +
.../documentation/io/built-in/hcatalog/index.html | 13 +
.../documentation/io/built-in/index.html | 1583 ++++++++++++++++-
.../documentation/io/built-in/parquet/index.html | 13 +
.../documentation/io/developing-io-java/index.html | 13 +
.../io/developing-io-overview/index.html | 13 +
.../io/developing-io-python/index.html | 13 +
.../documentation/io/testing/index.html | 13 +
.../{file-processing => bigqueryio}/index.html | 235 +--
.../documentation/patterns/custom-io/index.html | 13 +
.../patterns/custom-windows/index.html | 13 +
.../patterns/file-processing/index.html | 13 +
.../documentation/patterns/overview/index.html | 17 +
.../patterns/pipeline-options/index.html | 13 +
.../documentation/patterns/side-inputs/index.html | 13 +
.../pipelines/create-your-pipeline/index.html | 13 +
.../pipelines/design-your-pipeline/index.html | 13 +
.../pipelines/test-your-pipeline/index.html | 13 +
.../documentation/programming-guide/index.html | 13 +
.../resources/learning-resources/index.html | 13 +
.../resources/videos-and-podcasts/index.html | 13 +
.../documentation/runners/jstorm/index.html | 13 +
.../documentation/runtime/environments/index.html | 13 +
.../documentation/runtime/model/index.html | 13 +
.../runtime/sdk-harness-config/index.html | 13 +
.../aggregation/approximatequantiles/index.html | 13 +
.../java/aggregation/approximateunique/index.html | 13 +
.../java/aggregation/cogroupbykey/index.html | 13 +
.../transforms/java/aggregation/combine/index.html | 13 +
.../java/aggregation/combinewithcontext/index.html | 13 +
.../transforms/java/aggregation/count/index.html | 13 +
.../java/aggregation/distinct/index.html | 13 +
.../java/aggregation/groupbykey/index.html | 13 +
.../java/aggregation/groupintobatches/index.html | 13 +
.../java/aggregation/hllcount/index.html | 13 +
.../transforms/java/aggregation/latest/index.html | 13 +
.../transforms/java/aggregation/max/index.html | 13 +
.../transforms/java/aggregation/mean/index.html | 13 +
.../transforms/java/aggregation/min/index.html | 13 +
.../transforms/java/aggregation/sample/index.html | 13 +
.../transforms/java/aggregation/sum/index.html | 13 +
.../transforms/java/aggregation/top/index.html | 13 +
.../transforms/java/elementwise/filter/index.html | 13 +
.../java/elementwise/flatmapelements/index.html | 13 +
.../transforms/java/elementwise/keys/index.html | 13 +
.../transforms/java/elementwise/kvswap/index.html | 13 +
.../java/elementwise/mapelements/index.html | 13 +
.../transforms/java/elementwise/pardo/index.html | 13 +
.../java/elementwise/partition/index.html | 13 +
.../transforms/java/elementwise/regex/index.html | 13 +
.../transforms/java/elementwise/reify/index.html | 13 +
.../java/elementwise/tostring/index.html | 13 +
.../transforms/java/elementwise/values/index.html | 13 +
.../java/elementwise/withkeys/index.html | 13 +
.../java/elementwise/withtimestamps/index.html | 13 +
.../transforms/java/other/create/index.html | 13 +
.../transforms/java/other/flatten/index.html | 13 +
.../transforms/java/other/passert/index.html | 13 +
.../transforms/java/other/view/index.html | 13 +
.../transforms/java/other/window/index.html | 13 +
.../transforms/java/overview/index.html | 13 +
.../aggregation/approximatequantiles/index.html | 13 +
.../aggregation/approximateunique/index.html | 13 +
.../python/aggregation/cogroupbykey/index.html | 13 +
.../python/aggregation/combineglobally/index.html | 13 +
.../aggregation/combinewithcontext/index.html | 13 +
.../transforms/python/aggregation/count/index.html | 13 +
.../python/aggregation/distinct/index.html | 13 +
.../python/aggregation/groupbykey/index.html | 13 +
.../python/aggregation/groupintobatches/index.html | 13 +
.../python/aggregation/latest/index.html | 13 +
.../transforms/python/aggregation/max/index.html | 13 +
.../transforms/python/aggregation/mean/index.html | 13 +
.../transforms/python/aggregation/min/index.html | 13 +
.../python/aggregation/sample/index.html | 13 +
.../transforms/python/aggregation/sum/index.html | 13 +
.../transforms/python/aggregation/top/index.html | 13 +
.../python/elementwise/filter/index.html | 13 +
.../python/elementwise/flatmap/index.html | 13 +
.../transforms/python/elementwise/keys/index.html | 13 +
.../python/elementwise/kvswap/index.html | 13 +
.../transforms/python/elementwise/map/index.html | 13 +
.../transforms/python/elementwise/pardo/index.html | 13 +
.../python/elementwise/partition/index.html | 13 +
.../transforms/python/elementwise/regex/index.html | 13 +
.../transforms/python/elementwise/reify/index.html | 13 +
.../python/elementwise/tostring/index.html | 13 +
.../python/elementwise/values/index.html | 13 +
.../python/elementwise/withkeys/index.html | 13 +
.../python/elementwise/withtimestamps/index.html | 13 +
.../transforms/python/other/create/index.html | 13 +
.../transforms/python/other/flatten/index.html | 13 +
.../transforms/python/other/passert/index.html | 13 +
.../transforms/python/other/reshuffle/index.html | 13 +
.../transforms/python/other/view/index.html | 13 +
.../transforms/python/other/windowinto/index.html | 13 +
.../transforms/python/overview/index.html | 13 +
website/generated-content/sitemap.xml | 9 +-
101 files changed, 4531 insertions(+), 349 deletions(-)
diff --git a/website/generated-content/documentation/index.html b/website/generated-content/documentation/index.html
index 3a5f396..3ee711f 100644
--- a/website/generated-content/documentation/index.html
+++ b/website/generated-content/documentation/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/index.xml b/website/generated-content/documentation/index.xml
index 0d683f0..365d87a 100644
--- a/website/generated-content/documentation/index.xml
+++ b/website/generated-content/documentation/index.xml
@@ -8168,108 +8168,1675 @@ we want to batch ten seconds worth of events together in order to reduce the num
</item>
<item>
+ <title>Documentation: BigQuery patterns</title>
+ <link>/documentation/patterns/bigqueryio/</link>
+ <pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate>
+
+ <guid>/documentation/patterns/bigqueryio/</guid>
+ <description>
+
+
+ <!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1 id="google-bigquery-patterns">Google BigQuery patterns</h1>
+<p>The samples on this page show you common patterns for use with BigQueryIO.</p>
+
+
+<nav class="language-switcher">
+ <strong>Adapt for:</strong>
+ <ul>
+
+
+ <li data-type="language-java" class="active">Java SDK</li>
+
+
+
+
+
+
+ <li data-type="language-py">Python SDK</li>
+
+
+
+ </ul>
+</nav>
+<h2 id="bigqueryio-deadletter-pattern">BigQueryIO deadletter pattern</h2>
+<p>In production systems, it is useful to implement the deadletter pattern with BigQueryIO outputting any elements which had errors during processing by BigQueryIO into another PCollection for further processing.
+The samples below print the errors, but in a production system they can be sent to a deadletter table for later correction.</p>
+
+
+
+ <p class="language-java">When using <code>STREAMING_INSERTS</code> you can use the <code>WriteResult</code> object to access a <code>PCollection</code> with the <code>TableRows</code> that failed to be inserted into BigQuery.
+If you also set the <code>withExtendedErrorInfo</code> property , you will be able to access a <code>PCollection&lt;BigQueryInsertError&gt;</code> from the <code>WriteResult</code>. The <code>PCollection</code> will then include a reference to the table, the data row and the <code>InsertErrors</code>. Which errors are added to the deadletter queue is determined via the <code>InsertRetryPolicy</code>.</p>
+
+
+
+
+ <p class="language-py">In the result tuple you can access <code>FailedRows</code> to access the failed inserts.</p>
+
+
+
+
+
+
+
+ <div class=language-java>
+ <div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"> <span class="n">PipelineOptions</span> <span class="n">options</span> <span class="o">=</span>
+ <span class="n">PipelineOptionsFactory</span><span class="o">.</span><span class="na">fromArgs</span><span class="o">(</span><span class="n">args</span><span class="o">).</span><span class="na">withValidation</span><span class="o">().</span><span class="na">as</span><span class="o">(< [...]
+
+ <span class="n">Pipeline</span> <span class="n">p</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="o">.</span><span class="na">create</span><span class="o">(</span><span class="n">options</span><span class="o">);</span>
+
+ <span class="c1">// Create a bug by writing the 2nd value as null. The API will correctly
+</span><span class="c1"></span> <span class="c1">// throw an error when trying to insert a null value into a REQUIRED field.
+</span><span class="c1"></span> <span class="n">WriteResult</span> <span class="n">result</span> <span class="o">=</span>
+ <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Create</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">1</span><span class="o">,</span> <span class="n"> [...]
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">BigQueryIO</span><span class="o">.&lt;</span><span class="n">Integer</span><span class="o">&gt;</span><span class="n">write</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withSchema</span><span class="o">(</span>
+ <span class="k">new</span> <span class="n">TableSchema</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">setFields</span><span class="o">(</span>
+ <span class="n">ImmutableList</span><span class="o">.</span><span class="na">of</span><span class="o">(</span>
+ <span class="k">new</span> <span class="n">TableFieldSchema</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">setName</span><span class="o">(</span><span class="s">&#34;num&#34;</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">setType</span><span class="o">(</span><span class="s">&#34;INTEGER&#34;</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">setMode</span><span class="o">(</span><span class="s">&#34;REQUIRED&#34;</span><span class="o">))))</span>
+ <span class="o">.</span><span class="na">to</span><span class="o">(</span><span class="s">&#34;Test.dummyTable&#34;</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withFormatFunction</span><span class="o">(</span><span class="n">x</span> <span class="o">-&gt;</span> <span class="k">new</span> <span class="n">TableRow</span><span class="o">().</span><span class="na">set</span><span class="o">( [...]
+ <span class="o">.</span><span class="na">withFailedInsertRetryPolicy</span><span class="o">(</span><span class="n">InsertRetryPolicy</span><span class="o">.</span><span class="na">retryTransientErrors</span><span class="o">())</span>
+ <span class="c1">// Forcing the bounded pipeline to use streaming inserts
+</span><span class="c1"></span> <span class="o">.</span><span class="na">withMethod</span><span class="o">(</span><span class="n">BigQueryIO</span><span class="o">.</span><span class="na">Write</span><span class="o">.</span><span class="na">Method</span><span class="o">. [...]
+ <span class="c1">// set the withExtendedErrorInfo property.
+</span><span class="c1"></span> <span class="o">.</span><span class="na">withExtendedErrorInfo</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withCreateDisposition</span><span class="o">(</span><span class="n">BigQueryIO</span><span class="o">.</span><span class="na">Write</span><span class="o">.</span><span class="na">CreateDisposition</span><span class="o">.</span><span class="na [...]
+ <span class="o">.</span><span class="na">withWriteDisposition</span><span class="o">(</span><span class="n">BigQueryIO</span><span class="o">.</span><span class="na">Write</span><span class="o">.</span><span class="na">WriteDisposition</span><span class="o">.</span><span class="na&# [...]
+
+ <span class="n">result</span>
+ <span class="o">.</span><span class="na">getFailedInsertsWithErr</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">MapElements</span><span class="o">.</span><span class="na">into</span><span class="o">(</span><span class="n">TypeDescriptors</span><span class="o">.</span><span class="na">strings</span><span class="o">())</span>
+ <span class="o">.</span><span class="na">via</span><span class="o">(</span>
+ <span class="n">x</span> <span class="o">-&gt;</span> <span class="o">{</span>
+ <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">&#34; The table was &#34;</span> <span class="o">+</span> <span class="n">x</span><span class="o&# [...]
+ <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">&#34; The row was &#34;</span> <span class="o">+</span> <span class="n">x</span><span class="o" [...]
+ <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">&#34; The error was &#34;</span> <span class="o">+</span> <span class="n">x</span><span class="o&# [...]
+ <span class="k">return</span> <span class="s">&#34;&#34;</span><span class="o">;</span>
+ <span class="o">}));</span>
+ <span class="n">p</span><span class="o">.</span><span class="na">run</span><span class="o">();</span>
+
+ <span class="cm">/* Sample Output From the pipeline:
+</span><span class="cm"> &lt;p&gt;The table was GenericData{classInfo=[datasetId, projectId, tableId], {datasetId=Test,projectId=&lt;&gt;, tableId=dummyTable}}
+</span><span class="cm"> &lt;p&gt;The row was GenericData{classInfo=[f], {num=null}}
+</span><span class="cm"> &lt;p&gt;The error was GenericData{classInfo=[errors, index],{errors=[GenericData{classInfo=[debugInfo, location, message, reason], {debugInfo=,location=, message=Missing required field: Msg_0_CLOUD_QUERY_TABLE.num., reason=invalid}}],index=0}}
+</span><span class="cm"> */</span>
+ <span class="o">}</span></code></pre></div>
+ </div>
+
+
+
+
+
+
+
+
+
+ <div class=language-py>
+ <div class="highlight"><pre class="chroma"><code class="language-py" data-lang="py"> <span class="c1"># Create pipeline.</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="p">({</span><span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="p">[{</span><span class="s1">&#39;name&#39;</span><span class="p">:</span> <span class="s1">&#39;a&#39;</span><span class= [...]
+
+ <span class="n">p</span> <span class="o">=</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">()</span>
+
+ <span class="n">errors</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">p</span> <span class="o">|</span> <span class="s1">&#39;Data&#39;</span> <span class="o">&gt;&gt;</span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="mi">1</span><span class="p">,</spa [...]
+ <span class="o">|</span> <span class="s1">&#39;CreateBrokenData&#39;</span> <span class="o">&gt;&gt;</span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">src</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;a&#39;</span><span class="p">:</span> <span cla [...]
+ <span class="o">|</span> <span class="s1">&#39;WriteToBigQuery&#39;</span> <span class="o">&gt;&gt;</span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">WriteToBigQuery</span><span class="p">(</span>
+ <span class="s2">&#34;&lt;Your Project:Test.dummy_a_table&#34;</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span>
+ <span class="n">insert_retry_strategy</span><span class="o">=</span><span class="s1">&#39;RETRY_ON_TRANSIENT_ERROR&#39;</span><span class="p">,</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
+ <span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_APPEND&#39;</span><span class="p">))</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">errors</span><span class="p">[</span><span class="s1">&#39;FailedRows&#39;</span><span class="p">]</span>
+ <span class="o">|</span> <span class="s1">&#39;PrintErrors&#39;</span> <span class="o">&gt;&gt;</span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span><span class="k">lambda</span> <span class="n">err</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s2">&#34;Error Found {}&#34;</ [...]
+ </div>
+
+
+
+
+ </description>
+ </item>
+
+ <item>
<title>Documentation: Built-in I/O Transforms</title>
<link>/documentation/io/built-in/</link>
<pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate>
- <guid>/documentation/io/built-in/</guid>
- <description>
-
-
- <!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+ <guid>/documentation/io/built-in/</guid>
+ <description>
+
+
+ <!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1 id="built-in-io-transforms">Built-in I/O Transforms</h1>
+<p>This table contains the currently available I/O transforms.</p>
+<p>Consult the <a href="/documentation/programming-guide#pipeline-io">Programming Guide I/O section</a> for general usage instructions.</p>
+
+
+<nav class="language-switcher">
+ <strong>Adapt for:</strong>
+ <ul>
+
+
+ <li data-type="language-java" class="active">Java SDK</li>
+
+
+
+
+
+
+ <li data-type="language-py">Python SDK</li>
+
+
+
+
+
+
+ <li data-type="language-go">Go SDK</li>
+
+
+ </ul>
+</nav>
+
+
+
+
+
+
+
+
+<h3>File-based</h3>
+<p>These I/O connectors involve working with files.</p>
+<table class="table table-bordered">
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+ <tr class="language-java">
+ <td>
+ FileIO
+
+ </td>
+ <td>General-purpose transforms for working with files: listing files (matching), reading and writing.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/FileIO.html">org.apache.beam.sdk.io.FileIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ FileIO
+
+ </td>
+ <td>General-purpose transforms for working with files: listing files (matching), reading and writing.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.fileio.html">apache_beam.io.FileIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ AvroIO
+
+ </td>
+ <td>PTransforms for reading from and writing to <a href="https://avro.apache.org/">Avro</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/AvroIO.html">org.apache.beam.sdk.io.AvroIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ AvroIO
+
+ </td>
+ <td>PTransforms for reading from and writing to <a href="https://avro.apache.org/">Avro</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.avroio.html">apache_beam.io.avroio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ AvroIO
+
+ </td>
+ <td>PTransforms for reading from and writing to <a href="https://avro.apache.org/">Avro</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/avroio">github.com/apache/beam/sdks/go/pkg/beam/io/avroio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ TextIO
+
+ </td>
+ <td>PTransforms for reading and writing text files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/TextIO.html">org.apache.beam.sdk.io.TextIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ TextIO
+
+ </td>
+ <td>PTransforms for reading and writing text files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.textio.html">apache_beam.io.textio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ TextIO
+
+ </td>
+ <td>PTransforms for reading and writing text files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/textio">github.com/apache/beam/sdks/go/pkg/beam/io/textio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ TFRecordIO
+
+ </td>
+ <td>PTransforms for reading and writing <a href="https://www.tensorflow.org/tutorials/load_data/tfrecord">TensorFlow TFRecord</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/TFRecordIO.html">org.apache.beam.sdk.io.TFRecordIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ TFRecordIO
+
+ </td>
+ <td>PTransforms for reading and writing <a href="https://www.tensorflow.org/tutorials/load_data/tfrecord">TensorFlow TFRecord</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.tfrecordio.html">apache_beam.io.tfrecordio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ XmlIO
+
+ </td>
+ <td>Transforms for reading and writing XML files using <a href="https://www.oracle.com/technical-resources/articles/javase/jaxb.html">JAXB</a> mappers.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/2.3.0/org/apache/beam/sdk/io/xml/XmlIO.html">org.apache.beam.sdk.io.xml.XmlIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ TikaIO
+
+ </td>
+ <td>Transforms for parsing arbitrary files using <a href="https://tika.apache.org/">Apache Tika</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/2.3.0/org/apache/beam/sdk/io/tika/TikaIO.html">org.apache.beam.sdk.io.tika.TikaIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ ParquetIO
+ <a href="/documentation/io/built-in/parquet/"> (guide)</a>
+ </td>
+ <td>IO for reading from and writing to <a href="https://parquet.apache.org/">Parquet</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/parquet/ParquetIO.html">org.apache.beam.sdk.io.parquet.ParquetIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ ParquetIO
+ <a href="/documentation/io/built-in/parquet/"> (guide)</a>
+ </td>
+ <td>IO for reading from and writing to <a href="https://parquet.apache.org/">Parquet</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.parquetio.html">apache_beam.io.parquetio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ ThriftIO
+
+ </td>
+ <td>PTransforms for reading and writing files containing <a href="https://thrift.apache.org/">Thrift</a>-encoded data.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/thrift/ThriftIO.html">org.apache.beam.sdk.io.thrift.ThriftIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-py">
+ <td>
+ VcfIO
+
+ </td>
+ <td>A source for reading from <a href="https://samtools.github.io/hts-specs/VCFv4.2.pdf">VCF files</a> (version 4.x).</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.vcfio.html">apache_beam.io.vcfio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-py">
+ <td>
+ S3IO
+
+ </td>
+ <td>A source for reading from and writing to <a href="https://aws.amazon.com/s3/">Amazon S3</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.aws.s3io.html">apache_beam.io.aws.s3io</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-py">
+ <td>
+ GcsIO
+
+ </td>
+ <td>A source for reading from and writing to <a href="https://cloud.google.com/storage">Google Cloud Storage</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.gcsio.html">apache_beam.io.gcp.gcsio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+</table>
+
+<h3>FileSystem</h3>
+<p>Beam provides a File system interface that defines APIs for writing file systems agnostic code. Several I/O connectors are implemented as a FileSystem implementation.</p>
+<table class="table table-bordered">
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+ <tr class="language-java">
+ <td>
+ HadoopFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing <a href="https://hadoop.apache.org/">Hadoop</a> Distributed File System files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.html">org.apache.beam.sdk.io.hdfs.HadoopFileSystemRegistrar</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ HadoopFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing <a href="https://hadoop.apache.org/">Hadoop</a> Distributed File System files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.hadoopfilesystem.html">apache_beam.io.hadoopfilesystem</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ GcsFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for <a href="https://cloud.google.com/storage">Google Cloud Storage</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.html">org.apache.beam.sdk.extensions.gcp.storage.GcsFileSystemRegistrar</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ GcsFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for <a href="https://cloud.google.com/storage">Google Cloud Storage</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.gcsfilesystem.html">apache_beam.io.gcp.gcsfilesystem</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ GcsFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for <a href="https://cloud.google.com/storage">Google Cloud Storage</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/gcs">github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/gcs</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ LocalFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing files on disk.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/LocalFileSystemRegistrar.html">org.apache.beam.sdk.io.LocalFileSystemRegistrar</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ LocalFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing files on disk.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.localfilesystem.html">apache_beam.io.localfilesystem</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ LocalFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing files on disk.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/local">github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/local</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ S3FileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for <a href="https://aws.amazon.com/s3/">Amazon S3</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hdfs/package-summary.html">org.apache.beam.sdk.io.aws.s3.S3FileSystemRegistrar</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-go">
+ <td>
+ In-memory
+
+ </td>
+ <td><code>FileSystem</code> implementation in memory; useful for testing.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs">github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs</a></div>
+
+ </td>
+ </tr>
+
+
+
+</table>
+
+<h3>Messaging</h3>
+<p>These I/O connectors typically involve working with unbounded sources that come from messaging sources.</p>
+<table class="table table-bordered">
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+ <tr class="language-java">
+ <td>
+ KinesisIO
+
+ </td>
+ <td>PTransforms for reading from and writing to <a href="https://aws.amazon.com/kinesis/">Kinesis</a> streams.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/kinesis/KinesisIO.html">org.apache.beam.sdk.io.kinesis.KinesisIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ AmqpIO
+
+ </td>
+ <td>AMQP 1.0 protocol using the Apache QPid Proton-J library</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/amqp/AmqpIO.html">org.apache.beam.sdk.io.amqp.AmqpIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ KafkaIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://kafka.apache.org/">Apache Kafka</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/kafka/KafkaIO.html">org.apache.beam.sdk.io.kafka.KafkaIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ KafkaIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://kafka.apache.org/">Apache Kafka</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.external.kafka.html">apache_beam.io.external.kafka</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ PubSubIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://cloud.google.com/pubsub">Google Cloud Pub/Sub</a> streams.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.html">org.apache.beam.sdk.io.gcp.pubsub.PubsubIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ PubSubIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://cloud.google.com/pubsub">Google Cloud Pub/Sub</a> streams.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.pubsub.html">apache_beam.io.gcp.pubsub</a></div>
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.external.gcp.pubsub.html">apache_beam.io.external.gcp.pubsub</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ PubSubIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://cloud.google.com/pubsub">Google Cloud Pub/Sub</a> streams.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/pubsubio">github.com/apache/beam/sdks/go/pkg/beam/io/pubsubio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ JmsIO
+
+ </td>
+ <td>An unbounded source for <a href="https://www.oracle.com/java/technologies/java-message-service.html">JMS</a> destinations (queues or topics).</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/jms/JmsIO.html">org.apache.beam.sdk.io.jms.JmsIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ MqttIO
+
+ </td>
+ <td>An unbounded source for <a href="https://mqtt.org/">MQTT</a> broker.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/mqtt/MqttIO.html">org.apache.beam.sdk.io.mqtt.MqttIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ RabbitMqIO
+
+ </td>
+ <td>A IO to publish or consume messages with a RabbitMQ broker.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://github.com/apache/beam/blob/master/sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java">org.apache.beam.sdk.io.rabbitmq.RabbitMqIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ SqsIO
+
+ </td>
+ <td>An unbounded source for <a href="https://aws.amazon.com/sqs/">Amazon Simple Queue Service (SQS)</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws/sqs/SqsIO.html">org.apache.beam.sdk.io.aws.sqs.SqsIO</a></div>
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws2/sqs/SqsIO.html">org.apache.beam.sdk.io.aws2.sqs.SqsIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ SnsIO
+
+ </td>
+ <td>PTransforms for writing to <a href="https://aws.amazon.com/sns/">Amazon Simple Notification Service (SNS)</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws/sns/SnsIO.html">org.apache.beam.sdk.io.aws.sns.SnsIO</a></div>
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws2/sns/SnsIO.html">org.apache.beam.sdk.io.aws2.sns.SnsIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+</table>
+
+<h3>Database</h3>
+<p>These I/O connectors are used to connect to database systems.</p>
+<table class="table table-bordered">
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+ <tr class="language-java">
+ <td>
+ CassandraIO
+
+ </td>
+ <td>An IO to read from <a href="https://cassandra.apache.org/">Apache Cassandra</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/cassandra/CassandraIO.html">org.apache.beam.sdk.io.cassandra.CassandraIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ HadoopFormatIO
+ <a href="/documentation/io/built-in/hadoop/"> (guide)</a>
+ </td>
+ <td>Allows for reading data from any source or writing data to any sink which implements <a href="https://hadoop.apache.org/">Hadoop</a> InputFormat or OutputFormat.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIO.html">org.apache.beam.sdk.io.hadoop.format.HadoopFormatIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ HBaseIO
+
+ </td>
+ <td>A bounded source and sink for <a href="https://hbase.apache.org/">HBase</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hbase/HBaseIO.html">org.apache.beam.sdk.io.hbase.HBaseIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ HCatalogIO
+ <a href="/documentation/io/built-in/hcatalog/"> (guide)</a>
+ </td>
+ <td>HCatalog source supports reading of HCatRecord from a <a href="https://cwiki.apache.org/confluence/display/Hive/HCatalog">HCatalog</a>-managed source, for example <a href="https://hive.apache.org/">Hive</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hcatalog/HCatalogIO.html">org.apache.beam.sdk.io.hcatalog.HCatalogIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ KuduIO
+
+ </td>
+ <td>A bounded source and sink for (Kudu)[https://kudu.apache.org/].</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://github.com/apache/beam/blob/master/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduIO.java">org.apache.beam.sdk.io.kudu</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ SolrIO
+
+ </td>
+ <td>Transforms for reading and writing data from/to <a href="https://lucene.apache.org/solr/">Solr</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/solr/SolrIO.html">org.apache.beam.sdk.io.solr.SolrIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ ElasticsearchIO
+
+ </td>
+ <td>Transforms for reading and writing data from/to <a href="https://www.elastic.co/elasticsearch/">Elasticsearch</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.html">org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ BigQueryIO
+ <a href="/documentation/io/built-in/google-bigquery/"> (guide)</a>
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigquery">Google Cloud BigQuery</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.html">org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ BigQueryIO
+ <a href="/documentation/io/built-in/google-bigquery/"> (guide)</a>
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigquery">Google Cloud BigQuery</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.bigquery.html">apache_beam.io.gcp.bigquery</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ BigQueryIO
+ <a href="/documentation/io/built-in/google-bigquery/"> (guide)</a>
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigquery">Google Cloud BigQuery</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/bigqueryio">github.com/apache/beam/sdks/go/pkg/beam/io/bigqueryio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ BigTableIO
+
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigtable/">Google Cloud Bigtable</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.html">org.apache.beam.sdk.io.gcp.bigtable.BigtableIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ BigTableIO
+
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigtable/">Google Cloud Bigtable</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.bigtableio.html">apache_beam.io.gcp.bigtableio module</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ DatastoreIO
+
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/datastore">Google Cloud Datastore</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/datastore/DatastoreIO.html">org.apache.beam.sdk.io.gcp.datastore.DatastoreIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ DatastoreIO
+
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/datastore">Google Cloud Datastore</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.datastore.v1new.datastoreio.html">apache_beam.io.gcp.datastore.v1new.datastoreio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ SpannerIO
+
+ </td>
+ <td>Experimental Transforms for reading from and writing to <a href="https://cloud.google.com/spanner">Google Cloud Spanner</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.html">org.apache.beam.sdk.io.gcp.spanner.SpannerIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ JdbcIO
+
+ </td>
+ <td>IO to read and write data on <a href="https://docs.oracle.com/javase/tutorial/jdbc/basics/index.html">JDBC</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/jdbc/JdbcIO.html">org.apache.beam.sdk.io.jdbc.JdbcIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ MongoDbIO
+
+ </td>
+ <td>IO to read and write data on <a href="https://www.mongodb.com/">MongoDB</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/mongodb/MongoDbIO.html">org.apache.beam.sdk.io.mongodb.MongoDbIO</a></div>
+
+ </td>
+ </tr>
+
-http://www.apache.org/licenses/LICENSE-2.0
+
+
+ <tr class="language-py">
+ <td>
+ MongoDbIO
+
+ </td>
+ <td>IO to read and write data on <a href="https://www.mongodb.com/">MongoDB</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.mongodbio.html">apache_beam.io.mongodbio</a></div>
+
+ </td>
+ </tr>
+
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<h1 id="built-in-io-transforms">Built-in I/O Transforms</h1>
-<p>This table contains the currently available I/O transforms.</p>
-<p>Consult the <a href="/documentation/programming-guide#pipeline-io">Programming Guide I/O section</a> for general usage instructions, and see the javadoc/pydoc for the particular I/O transforms.</p>
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ MongoDbGridFSIO
+
+ </td>
+ <td>IO to read and write data on <a href="https://docs.mongodb.com/manual/core/gridfs/">MongoDB GridFS</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.html">org.apache.beam.sdk.io.mongodb.MongoDbGridFSIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ RedisIO
+
+ </td>
+ <td>An IO to manipulate a <a href="https://redis.io/">Redis</a> key/value database.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/redis/RedisIO.html">org.apache.beam.sdk.io.redis.RedisIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ DynamoDBIO
+
+ </td>
+ <td>Read from and write to <a href="https://aws.amazon.com/dynamodb/">Amazon DynamoDB</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIO.html">org.apache.beam.sdk.io.aws.dynamodb.DynamoDBIO</a></div>
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIO.html">org.apache.beam.sdk.io.aws2.dynamodb.DynamoDBIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ ClickHouseIO
+
+ </td>
+ <td>Transform for writing to <a href="https://clickhouse.tech/">ClickHouse</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.html">org.apache.beam.sdk.io.clickhouse.ClickHouseIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-go">
+ <td>
+ DatabaseIO
+
+ </td>
+ <td>Package databaseio provides transformations and utilities to interact with a generic database / SQL API.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/databaseio">github.com/apache/beam/sdks/go/pkg/beam/io/databaseio</a></div>
+
+ </td>
+ </tr>
+
+
+
+</table>
+
+<h3>Miscellaneous</h3>
+<p>Miscellaneous I/O sources.</p>
<table class="table table-bordered">
-<tr>
- <th>Language</th>
- <th>File-based</th>
- <th>Messaging</th>
- <th>Database</th>
-</tr>
-<tr>
- <td>Java</td>
- <td>
- <p>Beam Java supports Apache HDFS, Amazon S3, Google Cloud Storage, and local filesystems.</p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java">FileIO</a> (general-purpose reading, writing, and matching of files)</p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java">AvroIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java">TextIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java">TFRecordIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java">XmlIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java">TikaIO</a></p>
- <p><a href="/documentation/io/built-in/parquet">ParquetIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftIO.java">ThriftIO</a></p>
- </td>
- <td>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/kinesis">Amazon Kinesis</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/amqp">AMQP</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/kafka">Apache Kafka</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub">Google Cloud Pub/Sub</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/jms">JMS</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/mqtt">MQTT</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java">RabbitMqIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsIO.java">SqsIO</a></p>
- </td>
- <td>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/cassandra">Apache Cassandra</a></p>
- <p><a href="/documentation/io/built-in/hadoop/">Apache Hadoop Input/Output Format</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/hbase">Apache HBase</a></p>
- <p><a href="/documentation/io/built-in/hcatalog">Apache Hive (HCatalog)</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/kudu">Apache Kudu</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/solr">Apache Solr</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/elasticsearch">Elasticsearch (v2.x, v5.x, v6.x)</a></p>
- <p><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable">Google Cloud Bigtable</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore">Google Cloud Datastore</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner">Google Cloud Spanner</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/jdbc">JDBC</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/mongodb">MongoDB</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/redis">Redis</a></p>
- </td>
-</tr>
-<tr>
- <td>Python/Batch</td>
- <td>
- <p>Beam Python supports Apache HDFS, Amazon S3, Google Cloud Storage, and local filesystems.</p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/avroio.py">avroio</a></p>
- <p><a href="/documentation/io/built-in/parquet">parquetio.py</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/textio.py">textio</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/tfrecordio.py">tfrecordio</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/vcfio.py">vcfio</a></p>
- </td>
- <td>
- </td>
- <td>
- <p><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/python/apache_beam/io/gcp/datastore">Google Cloud Datastore</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/gcp/bigtableio.py">Google Cloud Bigtable</a> (Write)</p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/mongodbio.py">MongoDB</a></p>
- </td>
-</tr>
-<tr>
- <td>Python/Streaming</td>
- <td>
- </td>
- <td>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/gcp/pubsub.py">Google Cloud Pub/Sub</a></p>
- </td>
- <td>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/gcp/bigquery.py">Google BigQuery (sink only)</a></p>
- </td>
-</tr>
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+
+
+
+
+ <tr class="language-py">
+ <td>
+ FlinkStreamingImpulseSource
+
+ </td>
+ <td>A PTransform that provides an unbounded, streaming source of empty byte arrays. This can only be used with the Flink runner.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/2.20.0/apache_beam.io.flink.flink_streaming_impulse_source.html">apache_beam.io.flink.flink_streaming_impulse_source</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ GenerateSequence
+
+ </td>
+ <td>Generates a bounded or unbounded stream of integers.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/2.20.0/org/apache/beam/sdk/io/GenerateSequence.html">org.apache.beam.sdk.io.GenerateSequence</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ GenerateSequence
+
+ </td>
+ <td>Generates a bounded or unbounded stream of integers.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/2.20.0/apache_beam.io.external.generate_sequence.html">apache_beam.io.external.generate_sequence.GenerateSequence</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
</table>
+
<h1 id="in-progress-io-transforms">In-Progress I/O Transforms</h1>
<p>This table contains I/O transforms that are currently planned or in-progress. Status information can be found on the JIRA issue, or on the GitHub PR linked to by the JIRA issue (if there is one).</p>
<table class="table table-bordered">
@@ -14249,38 +15816,5 @@ assigns timestamps to all the elements of a collection</li>
</description>
</item>
- <item>
- <title>Documentation: Latest</title>
- <link>/documentation/transforms/python/aggregation/latest/</link>
- <pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate>
-
- <guid>/documentation/transforms/python/aggregation/latest/</guid>
- <description>
-
-
- <!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<h1 id="latest">Latest</h1>
-<h2 id="examples">Examples</h2>
-<p>See <a href="https://issues.apache.org/jira/browse/BEAM-7390">BEAM-7390</a> for updates.</p>
-<h2 id="related-transforms">Related transforms</h2>
-<ul>
-<li><a href="/documentation/transforms/python/aggregation/sample">Sample</a> to combine elements. takes samples of the elements in a collection.</li>
-</ul>
-
- </description>
- </item>
-
</channel>
</rss>
diff --git a/website/generated-content/documentation/io/built-in/google-bigquery/index.html b/website/generated-content/documentation/io/built-in/google-bigquery/index.html
index 92918c3..16af1cc 100644
--- a/website/generated-content/documentation/io/built-in/google-bigquery/index.html
+++ b/website/generated-content/documentation/io/built-in/google-bigquery/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/io/built-in/hadoop/index.html b/website/generated-content/documentation/io/built-in/hadoop/index.html
index 0326da8..cad2092 100644
--- a/website/generated-content/documentation/io/built-in/hadoop/index.html
+++ b/website/generated-content/documentation/io/built-in/hadoop/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/io/built-in/hcatalog/index.html b/website/generated-content/documentation/io/built-in/hcatalog/index.html
index c86daf7..947c31f 100644
--- a/website/generated-content/documentation/io/built-in/hcatalog/index.html
+++ b/website/generated-content/documentation/io/built-in/hcatalog/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/io/built-in/index.html b/website/generated-content/documentation/io/built-in/index.html
index 13b8af7..bf81147 100644
--- a/website/generated-content/documentation/io/built-in/index.html
+++ b/website/generated-content/documentation/io/built-in/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
@@ -456,85 +469,1501 @@ limitations under the License.
-->
<h1 id="built-in-io-transforms">Built-in I/O Transforms</h1>
<p>This table contains the currently available I/O transforms.</p>
-<p>Consult the <a href="/documentation/programming-guide#pipeline-io">Programming Guide I/O section</a> for general usage instructions, and see the javadoc/pydoc for the particular I/O transforms.</p>
+<p>Consult the <a href="/documentation/programming-guide#pipeline-io">Programming Guide I/O section</a> for general usage instructions.</p>
+
+
+<nav class="language-switcher">
+ <strong>Adapt for:</strong>
+ <ul>
+
+
+ <li data-type="language-java" class="active">Java SDK</li>
+
+
+
+
+
+
+ <li data-type="language-py">Python SDK</li>
+
+
+
+
+
+
+ <li data-type="language-go">Go SDK</li>
+
+
+ </ul>
+</nav>
+
+
+
+
+
+
+
+
+<h3>File-based</h3>
+<p>These I/O connectors involve working with files.</p>
<table class="table table-bordered">
-<tr>
- <th>Language</th>
- <th>File-based</th>
- <th>Messaging</th>
- <th>Database</th>
-</tr>
-<tr>
- <td>Java</td>
- <td>
- <p>Beam Java supports Apache HDFS, Amazon S3, Google Cloud Storage, and local filesystems.</p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java">FileIO</a> (general-purpose reading, writing, and matching of files)</p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java">AvroIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java">TextIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java">TFRecordIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java">XmlIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java">TikaIO</a></p>
- <p><a href="/documentation/io/built-in/parquet">ParquetIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftIO.java">ThriftIO</a></p>
- </td>
- <td>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/kinesis">Amazon Kinesis</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/amqp">AMQP</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/kafka">Apache Kafka</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub">Google Cloud Pub/Sub</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/jms">JMS</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/mqtt">MQTT</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java">RabbitMqIO</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsIO.java">SqsIO</a></p>
- </td>
- <td>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/cassandra">Apache Cassandra</a></p>
- <p><a href="/documentation/io/built-in/hadoop/">Apache Hadoop Input/Output Format</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/hbase">Apache HBase</a></p>
- <p><a href="/documentation/io/built-in/hcatalog">Apache Hive (HCatalog)</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/kudu">Apache Kudu</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/solr">Apache Solr</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/elasticsearch">Elasticsearch (v2.x, v5.x, v6.x)</a></p>
- <p><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable">Google Cloud Bigtable</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore">Google Cloud Datastore</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner">Google Cloud Spanner</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/jdbc">JDBC</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/mongodb">MongoDB</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/java/io/redis">Redis</a></p>
- </td>
-</tr>
-<tr>
- <td>Python/Batch</td>
- <td>
- <p>Beam Python supports Apache HDFS, Amazon S3, Google Cloud Storage, and local filesystems.</p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/avroio.py">avroio</a></p>
- <p><a href="/documentation/io/built-in/parquet">parquetio.py</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/textio.py">textio</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/tfrecordio.py">tfrecordio</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/vcfio.py">vcfio</a></p>
- </td>
- <td>
- </td>
- <td>
- <p><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery</a></p>
- <p><a href="https://github.com/apache/beam/tree/master/sdks/python/apache_beam/io/gcp/datastore">Google Cloud Datastore</a></p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/gcp/bigtableio.py">Google Cloud Bigtable</a> (Write)</p>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/mongodbio.py">MongoDB</a></p>
- </td>
-</tr>
-<tr>
- <td>Python/Streaming</td>
- <td>
- </td>
- <td>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/gcp/pubsub.py">Google Cloud Pub/Sub</a></p>
- </td>
- <td>
- <p><a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/gcp/bigquery.py">Google BigQuery (sink only)</a></p>
- </td>
-</tr>
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+ <tr class="language-java">
+ <td>
+ FileIO
+
+ </td>
+ <td>General-purpose transforms for working with files: listing files (matching), reading and writing.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/FileIO.html">org.apache.beam.sdk.io.FileIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ FileIO
+
+ </td>
+ <td>General-purpose transforms for working with files: listing files (matching), reading and writing.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.fileio.html">apache_beam.io.FileIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ AvroIO
+
+ </td>
+ <td>PTransforms for reading from and writing to <a href="https://avro.apache.org/">Avro</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/AvroIO.html">org.apache.beam.sdk.io.AvroIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ AvroIO
+
+ </td>
+ <td>PTransforms for reading from and writing to <a href="https://avro.apache.org/">Avro</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.avroio.html">apache_beam.io.avroio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ AvroIO
+
+ </td>
+ <td>PTransforms for reading from and writing to <a href="https://avro.apache.org/">Avro</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/avroio">github.com/apache/beam/sdks/go/pkg/beam/io/avroio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ TextIO
+
+ </td>
+ <td>PTransforms for reading and writing text files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/TextIO.html">org.apache.beam.sdk.io.TextIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ TextIO
+
+ </td>
+ <td>PTransforms for reading and writing text files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.textio.html">apache_beam.io.textio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ TextIO
+
+ </td>
+ <td>PTransforms for reading and writing text files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/textio">github.com/apache/beam/sdks/go/pkg/beam/io/textio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ TFRecordIO
+
+ </td>
+ <td>PTransforms for reading and writing <a href="https://www.tensorflow.org/tutorials/load_data/tfrecord">TensorFlow TFRecord</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/TFRecordIO.html">org.apache.beam.sdk.io.TFRecordIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ TFRecordIO
+
+ </td>
+ <td>PTransforms for reading and writing <a href="https://www.tensorflow.org/tutorials/load_data/tfrecord">TensorFlow TFRecord</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.tfrecordio.html">apache_beam.io.tfrecordio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ XmlIO
+
+ </td>
+ <td>Transforms for reading and writing XML files using <a href="https://www.oracle.com/technical-resources/articles/javase/jaxb.html">JAXB</a> mappers.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/2.3.0/org/apache/beam/sdk/io/xml/XmlIO.html">org.apache.beam.sdk.io.xml.XmlIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ TikaIO
+
+ </td>
+ <td>Transforms for parsing arbitrary files using <a href="https://tika.apache.org/">Apache Tika</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/2.3.0/org/apache/beam/sdk/io/tika/TikaIO.html">org.apache.beam.sdk.io.tika.TikaIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ ParquetIO
+ <a href="/documentation/io/built-in/parquet/"> (guide)</a>
+ </td>
+ <td>IO for reading from and writing to <a href="https://parquet.apache.org/">Parquet</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/parquet/ParquetIO.html">org.apache.beam.sdk.io.parquet.ParquetIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ ParquetIO
+ <a href="/documentation/io/built-in/parquet/"> (guide)</a>
+ </td>
+ <td>IO for reading from and writing to <a href="https://parquet.apache.org/">Parquet</a> files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.parquetio.html">apache_beam.io.parquetio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ ThriftIO
+
+ </td>
+ <td>PTransforms for reading and writing files containing <a href="https://thrift.apache.org/">Thrift</a>-encoded data.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/thrift/ThriftIO.html">org.apache.beam.sdk.io.thrift.ThriftIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-py">
+ <td>
+ VcfIO
+
+ </td>
+ <td>A source for reading from <a href="https://samtools.github.io/hts-specs/VCFv4.2.pdf">VCF files</a> (version 4.x).</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.vcfio.html">apache_beam.io.vcfio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-py">
+ <td>
+ S3IO
+
+ </td>
+ <td>A source for reading from and writing to <a href="https://aws.amazon.com/s3/">Amazon S3</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.aws.s3io.html">apache_beam.io.aws.s3io</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-py">
+ <td>
+ GcsIO
+
+ </td>
+ <td>A source for reading from and writing to <a href="https://cloud.google.com/storage">Google Cloud Storage</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.gcsio.html">apache_beam.io.gcp.gcsio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
</table>
+
+<h3>FileSystem</h3>
+<p>Beam provides a File system interface that defines APIs for writing file systems agnostic code. Several I/O connectors are implemented as a FileSystem implementation.</p>
+<table class="table table-bordered">
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+ <tr class="language-java">
+ <td>
+ HadoopFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing <a href="https://hadoop.apache.org/">Hadoop</a> Distributed File System files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hdfs/HadoopFileSystemRegistrar.html">org.apache.beam.sdk.io.hdfs.HadoopFileSystemRegistrar</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ HadoopFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing <a href="https://hadoop.apache.org/">Hadoop</a> Distributed File System files.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.hadoopfilesystem.html">apache_beam.io.hadoopfilesystem</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ GcsFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for <a href="https://cloud.google.com/storage">Google Cloud Storage</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/extensions/gcp/storage/GcsFileSystemRegistrar.html">org.apache.beam.sdk.extensions.gcp.storage.GcsFileSystemRegistrar</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ GcsFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for <a href="https://cloud.google.com/storage">Google Cloud Storage</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.gcsfilesystem.html">apache_beam.io.gcp.gcsfilesystem</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ GcsFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for <a href="https://cloud.google.com/storage">Google Cloud Storage</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/gcs">github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/gcs</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ LocalFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing files on disk.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/LocalFileSystemRegistrar.html">org.apache.beam.sdk.io.LocalFileSystemRegistrar</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ LocalFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing files on disk.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.localfilesystem.html">apache_beam.io.localfilesystem</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ LocalFileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for accessing files on disk.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/local">github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/local</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ S3FileSystem
+
+ </td>
+ <td><code>FileSystem</code> implementation for <a href="https://aws.amazon.com/s3/">Amazon S3</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hdfs/package-summary.html">org.apache.beam.sdk.io.aws.s3.S3FileSystemRegistrar</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-go">
+ <td>
+ In-memory
+
+ </td>
+ <td><code>FileSystem</code> implementation in memory; useful for testing.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs">github.com/apache/beam/sdks/go/pkg/beam/io/filesystem/memfs</a></div>
+
+ </td>
+ </tr>
+
+
+
+</table>
+
+<h3>Messaging</h3>
+<p>These I/O connectors typically involve working with unbounded sources that come from messaging sources.</p>
+<table class="table table-bordered">
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+ <tr class="language-java">
+ <td>
+ KinesisIO
+
+ </td>
+ <td>PTransforms for reading from and writing to <a href="https://aws.amazon.com/kinesis/">Kinesis</a> streams.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/kinesis/KinesisIO.html">org.apache.beam.sdk.io.kinesis.KinesisIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ AmqpIO
+
+ </td>
+ <td>AMQP 1.0 protocol using the Apache QPid Proton-J library</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/amqp/AmqpIO.html">org.apache.beam.sdk.io.amqp.AmqpIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ KafkaIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://kafka.apache.org/">Apache Kafka</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/kafka/KafkaIO.html">org.apache.beam.sdk.io.kafka.KafkaIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ KafkaIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://kafka.apache.org/">Apache Kafka</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.external.kafka.html">apache_beam.io.external.kafka</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ PubSubIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://cloud.google.com/pubsub">Google Cloud Pub/Sub</a> streams.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.html">org.apache.beam.sdk.io.gcp.pubsub.PubsubIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ PubSubIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://cloud.google.com/pubsub">Google Cloud Pub/Sub</a> streams.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.pubsub.html">apache_beam.io.gcp.pubsub</a></div>
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.external.gcp.pubsub.html">apache_beam.io.external.gcp.pubsub</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ PubSubIO
+
+ </td>
+ <td>Read and Write PTransforms for <a href="https://cloud.google.com/pubsub">Google Cloud Pub/Sub</a> streams.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/pubsubio">github.com/apache/beam/sdks/go/pkg/beam/io/pubsubio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ JmsIO
+
+ </td>
+ <td>An unbounded source for <a href="https://www.oracle.com/java/technologies/java-message-service.html">JMS</a> destinations (queues or topics).</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/jms/JmsIO.html">org.apache.beam.sdk.io.jms.JmsIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ MqttIO
+
+ </td>
+ <td>An unbounded source for <a href="https://mqtt.org/">MQTT</a> broker.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/mqtt/MqttIO.html">org.apache.beam.sdk.io.mqtt.MqttIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ RabbitMqIO
+
+ </td>
+ <td>A IO to publish or consume messages with a RabbitMQ broker.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://github.com/apache/beam/blob/master/sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java">org.apache.beam.sdk.io.rabbitmq.RabbitMqIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ SqsIO
+
+ </td>
+ <td>An unbounded source for <a href="https://aws.amazon.com/sqs/">Amazon Simple Queue Service (SQS)</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws/sqs/SqsIO.html">org.apache.beam.sdk.io.aws.sqs.SqsIO</a></div>
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws2/sqs/SqsIO.html">org.apache.beam.sdk.io.aws2.sqs.SqsIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ SnsIO
+
+ </td>
+ <td>PTransforms for writing to <a href="https://aws.amazon.com/sns/">Amazon Simple Notification Service (SNS)</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws/sns/SnsIO.html">org.apache.beam.sdk.io.aws.sns.SnsIO</a></div>
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws2/sns/SnsIO.html">org.apache.beam.sdk.io.aws2.sns.SnsIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+</table>
+
+<h3>Database</h3>
+<p>These I/O connectors are used to connect to database systems.</p>
+<table class="table table-bordered">
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+ <tr class="language-java">
+ <td>
+ CassandraIO
+
+ </td>
+ <td>An IO to read from <a href="https://cassandra.apache.org/">Apache Cassandra</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/cassandra/CassandraIO.html">org.apache.beam.sdk.io.cassandra.CassandraIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ HadoopFormatIO
+ <a href="/documentation/io/built-in/hadoop/"> (guide)</a>
+ </td>
+ <td>Allows for reading data from any source or writing data to any sink which implements <a href="https://hadoop.apache.org/">Hadoop</a> InputFormat or OutputFormat.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hadoop/format/HadoopFormatIO.html">org.apache.beam.sdk.io.hadoop.format.HadoopFormatIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ HBaseIO
+
+ </td>
+ <td>A bounded source and sink for <a href="https://hbase.apache.org/">HBase</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hbase/HBaseIO.html">org.apache.beam.sdk.io.hbase.HBaseIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ HCatalogIO
+ <a href="/documentation/io/built-in/hcatalog/"> (guide)</a>
+ </td>
+ <td>HCatalog source supports reading of HCatRecord from a <a href="https://cwiki.apache.org/confluence/display/Hive/HCatalog">HCatalog</a>-managed source, for example <a href="https://hive.apache.org/">Hive</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/hcatalog/HCatalogIO.html">org.apache.beam.sdk.io.hcatalog.HCatalogIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ KuduIO
+
+ </td>
+ <td>A bounded source and sink for (Kudu)[https://kudu.apache.org/].</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://github.com/apache/beam/blob/master/sdks/java/io/kudu/src/main/java/org/apache/beam/sdk/io/kudu/KuduIO.java">org.apache.beam.sdk.io.kudu</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ SolrIO
+
+ </td>
+ <td>Transforms for reading and writing data from/to <a href="https://lucene.apache.org/solr/">Solr</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/solr/SolrIO.html">org.apache.beam.sdk.io.solr.SolrIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ ElasticsearchIO
+
+ </td>
+ <td>Transforms for reading and writing data from/to <a href="https://www.elastic.co/elasticsearch/">Elasticsearch</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIO.html">org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ BigQueryIO
+ <a href="/documentation/io/built-in/google-bigquery/"> (guide)</a>
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigquery">Google Cloud BigQuery</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.html">org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ BigQueryIO
+ <a href="/documentation/io/built-in/google-bigquery/"> (guide)</a>
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigquery">Google Cloud BigQuery</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.bigquery.html">apache_beam.io.gcp.bigquery</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-go">
+ <td>
+ BigQueryIO
+ <a href="/documentation/io/built-in/google-bigquery/"> (guide)</a>
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigquery">Google Cloud BigQuery</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/bigqueryio">github.com/apache/beam/sdks/go/pkg/beam/io/bigqueryio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ BigTableIO
+
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigtable/">Google Cloud Bigtable</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.html">org.apache.beam.sdk.io.gcp.bigtable.BigtableIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ BigTableIO
+
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/bigtable/">Google Cloud Bigtable</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.bigtableio.html">apache_beam.io.gcp.bigtableio module</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ DatastoreIO
+
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/datastore">Google Cloud Datastore</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/datastore/DatastoreIO.html">org.apache.beam.sdk.io.gcp.datastore.DatastoreIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ DatastoreIO
+
+ </td>
+ <td>Read from and write to <a href="https://cloud.google.com/datastore">Google Cloud Datastore</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.datastore.v1new.datastoreio.html">apache_beam.io.gcp.datastore.v1new.datastoreio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ SpannerIO
+
+ </td>
+ <td>Experimental Transforms for reading from and writing to <a href="https://cloud.google.com/spanner">Google Cloud Spanner</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.html">org.apache.beam.sdk.io.gcp.spanner.SpannerIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ JdbcIO
+
+ </td>
+ <td>IO to read and write data on <a href="https://docs.oracle.com/javase/tutorial/jdbc/basics/index.html">JDBC</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/jdbc/JdbcIO.html">org.apache.beam.sdk.io.jdbc.JdbcIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ MongoDbIO
+
+ </td>
+ <td>IO to read and write data on <a href="https://www.mongodb.com/">MongoDB</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/mongodb/MongoDbIO.html">org.apache.beam.sdk.io.mongodb.MongoDbIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ MongoDbIO
+
+ </td>
+ <td>IO to read and write data on <a href="https://www.mongodb.com/">MongoDB</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.mongodbio.html">apache_beam.io.mongodbio</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ MongoDbGridFSIO
+
+ </td>
+ <td>IO to read and write data on <a href="https://docs.mongodb.com/manual/core/gridfs/">MongoDB GridFS</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.html">org.apache.beam.sdk.io.mongodb.MongoDbGridFSIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ RedisIO
+
+ </td>
+ <td>An IO to manipulate a <a href="https://redis.io/">Redis</a> key/value database.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/redis/RedisIO.html">org.apache.beam.sdk.io.redis.RedisIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ DynamoDBIO
+
+ </td>
+ <td>Read from and write to <a href="https://aws.amazon.com/dynamodb/">Amazon DynamoDB</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws/dynamodb/DynamoDBIO.html">org.apache.beam.sdk.io.aws.dynamodb.DynamoDBIO</a></div>
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/aws2/dynamodb/DynamoDBIO.html">org.apache.beam.sdk.io.aws2.dynamodb.DynamoDBIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ ClickHouseIO
+
+ </td>
+ <td>Transform for writing to <a href="https://clickhouse.tech/">ClickHouse</a>.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.html">org.apache.beam.sdk.io.clickhouse.ClickHouseIO</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <tr class="language-go">
+ <td>
+ DatabaseIO
+
+ </td>
+ <td>Package databaseio provides transformations and utilities to interact with a generic database / SQL API.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://godoc.org/github.com/apache/beam/sdks/go/pkg/beam/io/databaseio">github.com/apache/beam/sdks/go/pkg/beam/io/databaseio</a></div>
+
+ </td>
+ </tr>
+
+
+
+</table>
+
+<h3>Miscellaneous</h3>
+<p>Miscellaneous I/O sources.</p>
+<table class="table table-bordered">
+ <tr>
+ <th>Name</th>
+ <th>Description</th>
+ <th>
+ <span class="language-java">Javadoc</span>
+ <span class="language-py">pydoc</span>
+ <span class="language-go">Godoc</span>
+ </th>
+ </tr>
+
+
+
+
+
+
+
+ <tr class="language-py">
+ <td>
+ FlinkStreamingImpulseSource
+
+ </td>
+ <td>A PTransform that provides an unbounded, streaming source of empty byte arrays. This can only be used with the Flink runner.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/2.20.0/apache_beam.io.flink.flink_streaming_impulse_source.html">apache_beam.io.flink.flink_streaming_impulse_source</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+
+
+ <tr class="language-java">
+ <td>
+ GenerateSequence
+
+ </td>
+ <td>Generates a bounded or unbounded stream of integers.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/javadoc/2.20.0/org/apache/beam/sdk/io/GenerateSequence.html">org.apache.beam.sdk.io.GenerateSequence</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+ <tr class="language-py">
+ <td>
+ GenerateSequence
+
+ </td>
+ <td>Generates a bounded or unbounded stream of integers.</td>
+ <td style="word-break: break-all;">
+
+ <div><a target="_blank" href="https://beam.apache.org/releases/pydoc/2.20.0/apache_beam.io.external.generate_sequence.html">apache_beam.io.external.generate_sequence.GenerateSequence</a></div>
+
+ </td>
+ </tr>
+
+
+
+
+
+
+
+</table>
+
<h1 id="in-progress-io-transforms">In-Progress I/O Transforms</h1>
<p>This table contains I/O transforms that are currently planned or in-progress. Status information can be found on the JIRA issue, or on the GitHub PR linked to by the JIRA issue (if there is one).</p>
<table class="table table-bordered">
diff --git a/website/generated-content/documentation/io/built-in/parquet/index.html b/website/generated-content/documentation/io/built-in/parquet/index.html
index e84c8aa..11e810c 100644
--- a/website/generated-content/documentation/io/built-in/parquet/index.html
+++ b/website/generated-content/documentation/io/built-in/parquet/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/io/developing-io-java/index.html b/website/generated-content/documentation/io/developing-io-java/index.html
index c6b4116..b99d99c 100644
--- a/website/generated-content/documentation/io/developing-io-java/index.html
+++ b/website/generated-content/documentation/io/developing-io-java/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/io/developing-io-overview/index.html b/website/generated-content/documentation/io/developing-io-overview/index.html
index 2a76dae..c9c2300 100644
--- a/website/generated-content/documentation/io/developing-io-overview/index.html
+++ b/website/generated-content/documentation/io/developing-io-overview/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/io/developing-io-python/index.html b/website/generated-content/documentation/io/developing-io-python/index.html
index 6818d57..8a21733 100644
--- a/website/generated-content/documentation/io/developing-io-python/index.html
+++ b/website/generated-content/documentation/io/developing-io-python/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/io/testing/index.html b/website/generated-content/documentation/io/testing/index.html
index 84cfd09..827c50f 100644
--- a/website/generated-content/documentation/io/testing/index.html
+++ b/website/generated-content/documentation/io/testing/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/patterns/file-processing/index.html b/website/generated-content/documentation/patterns/bigqueryio/index.html
similarity index 66%
copy from website/generated-content/documentation/patterns/file-processing/index.html
copy to website/generated-content/documentation/patterns/bigqueryio/index.html
index 37d691f..7f22010 100644
--- a/website/generated-content/documentation/patterns/file-processing/index.html
+++ b/website/generated-content/documentation/patterns/bigqueryio/index.html
@@ -9,7 +9,7 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>File processing patterns</title>
+<title>BigQuery patterns</title>
<meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a [...]
<link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400" rel="stylesheet">
@@ -34,7 +34,7 @@
<script src="/js/page-nav.js"></script>
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="/feed.xml">
-<link rel="canonical" href="/documentation/patterns/file-processing/" data-proofer-ignore>
+<link rel="canonical" href="/documentation/patterns/bigqueryio/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.1/css/all.css" integrity="sha384-5sAR7xN1Nv6T6+dT2mhtzEpVJvfS3NScPQTrOxhwjIuvcA67KV2R5Jz6kr4abQsz" crossorigin="anonymous">
@@ -121,7 +121,7 @@
</li>
<li>
- <a href="https://github.com/apache/beam/edit/master/website/www/site/content/en/documentation/patterns/file-processing.md" data-proofer-ignore>
+ <a href="https://github.com/apache/beam/edit/master/website/www/site/content/en/documentation/patterns/bigqueryio.md" data-proofer-ignore>
<i class="far fa-edit fa-lg" alt="Edit on GitHub" title="Edit on GitHub"></i>
</a>
</li>
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
@@ -439,14 +452,7 @@
<nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
<nav id="TableOfContents">
<ul>
- <li><a href="#processing-files-as-they-arrive">Processing files as they arrive</a>
- <ul>
- <li><a href="#continuous-read-mode">Continuous read mode</a></li>
- <li><a href="#stream-processing-triggered-from-external-source">Stream processing triggered from external source</a></li>
- <li><a href="#batch-processing-triggered-from-external-source">Batch processing triggered from external source</a></li>
- </ul>
- </li>
- <li><a href="#accessing-filenames">Accessing filenames</a></li>
+ <li><a href="#bigqueryio-deadletter-pattern">BigQueryIO deadletter pattern</a></li>
</ul>
</nav>
</nav>
@@ -465,8 +471,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
-<h1 id="file-processing-patterns">File processing patterns</h1>
-<p>This page describes common file processing tasks. For more information on file-based I/O, see <a href="/documentation/programming-guide/#pipeline-io">Pipeline I/O</a> and <a href="/documentation/programming-guide/#file-based-data">File-based input and output data</a>.</p>
+<h1 id="google-bigquery-patterns">Google BigQuery patterns</h1>
+<p>The samples on this page show you common patterns for use with BigQueryIO.</p>
<nav class="language-switcher">
@@ -487,121 +493,19 @@ limitations under the License.
</ul>
</nav>
-<h2 id="processing-files-as-they-arrive">Processing files as they arrive</h2>
-<p>This section shows you how to process files as they arrive in your file system or object store (like Google Cloud Storage). You can continuously read files or trigger stream and processing pipelines when a file arrives.</p>
-<h3 id="continuous-read-mode">Continuous read mode</h3>
-
-
-
- <p class="language-java">You can use <code>FileIO</code> or <code>TextIO</code> to continuously read the source for new files.</p>
-
-
-
-
- <p class="language-java">Use the <a href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/FileIO.html"><code>FileIO</code></a> class to continuously watch a single file pattern. The following example matches a file pattern repeatedly every 30 seconds, continuously returns new matched files as an unbounded <code>PCollection<Metadata></code>, and stops if no new files appear for one hour:</p>
-
-
-
-
-
-
-
- <div class=language-java>
- <div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"><span class="c1">// This produces PCollection<MatchResult.Metadata>
-</span><span class="c1"></span><span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span>
- <span class="n">FileIO</span><span class="o">.</span><span class="na">match</span><span class="o">()</span>
- <span class="o">.</span><span class="na">filepattern</span><span class="o">(</span><span class="s">"..."</span><span class="o">)</span>
- <span class="o">.</span><span class="na">continuously</span><span class="o">(</span>
- <span class="n">Duration</span><span class="o">.</span><span class="na">standardSeconds</span><span class="o">(</span><span class="n">30</span><span class="o">),</span>
- <span class="n">Watch</span><span class="o">.</span><span class="na">Growth</span><span class="o">.</span><span class="na">afterTimeSinceNewOutput</span><span class="o">(</span><span class="n">Duration</span><span class="o">.</span><span class="na">standardHours</span><span class="o">(</span><span class="n">1</span><span class="o">))));</span></code></pre></div>
- </div>
-
-
-
-
-
-
- <p class="language-java">The <a href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/TextIO.html"><code>TextIO</code></a> class <code>watchForNewFiles</code> property streams new file matches.</p>
-
-
-
-
-
-
-
- <div class=language-java>
- <div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"><span class="c1">// This produces PCollection<String>
-</span><span class="c1"></span><span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span>
- <span class="n">TextIO</span><span class="o">.</span><span class="na">read</span><span class="o">()</span>
- <span class="o">.</span><span class="na">from</span><span class="o">(</span><span class="s">"<path-to-files>/*"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">watchForNewFiles</span><span class="o">(</span>
- <span class="c1">// Check for new files every minute.
-</span><span class="c1"></span> <span class="n">Duration</span><span class="o">.</span><span class="na">standardMinutes</span><span class="o">(</span><span class="n">1</span><span class="o">),</span>
- <span class="c1">// Stop watching the file pattern if no new files appear for an hour.
-</span><span class="c1"></span> <span class="n">Watch</span><span class="o">.</span><span class="na">Growth</span><span class="o">.</span><span class="na">afterTimeSinceNewOutput</span><span class="o">(</span><span class="n">Duration</span><span class="o">.</span><span class="na">standardHours</span><span class="o">(</span><span class="n">1</span><span class="o">))));</span></code></pre></div>
- </div>
-
-
-
-
-
-
- <p class="language-java">Some runners may retain file lists during updates, but file lists don’t persist when you restart a pipeline. You can save file lists by:</p>
-
-
-
-
- <p class="language-java"><ul>
-<li>Storing processed filenames in an external file and deduplicating the lists at the next transform</li>
-<li>Adding timestamps to filenames, writing a glob pattern to pull in only new files, and matching the pattern when the pipeline restarts</li>
-</ul></p>
-
-
-
-
- <p class="language-py">The continuous-read option is not available for Python.</p>
-
-<h3 id="stream-processing-triggered-from-external-source">Stream processing triggered from external source</h3>
-<p>A streaming pipeline can process data from an unbounded source. For example, to trigger stream processing with Google Cloud Pub/Sub:</p>
-<ol>
-<li>Use an external process to detect when new files arrive.</li>
-<li>Send a Google Cloud Pub/Sub message with a URI to the file.</li>
-<li>Access the URI from a <code>DoFn</code> that follows the Google Cloud Pub/Sub source.</li>
-<li>Process the file.</li>
-</ol>
-<h3 id="batch-processing-triggered-from-external-source">Batch processing triggered from external source</h3>
-<p>To start or schedule a batch pipeline job when a file arrives, write the triggering event in the source file itself. This has the most latency because the pipeline must initialize before processing. It’s best suited for low-frequency, large, file-size updates.</p>
-<h2 id="accessing-filenames">Accessing filenames</h2>
-
-
-
- <p class="language-java">Use the <code>FileIO</code> class to read filenames in a pipeline job. <code>FileIO</code> returns a <code>PCollection<ReadableFile></code> object, and the <code>ReadableFile</code> instance contains the filename.</p>
-
-
-
-
- <p class="language-java">To access filenames:</p>
-
-
-
-
- <p class="language-java"><ol>
-<li>Create a <code>ReadableFile</code> instance with <code>FileIO</code>. <code>FileIO</code> returns a <code>PCollection<ReadableFile></code> object. The <code>ReadableFile</code> class contains the filename.</li>
-<li>Call the <code>readFullyAsUTF8String()</code> method to read the file into memory and return the filename as a <code>String</code> object. If memory is limited, you can use utility classes like <a href="https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/FileSystems.html"><code>FileSystems</code></a> to work directly with the file.</li>
-</ol></p>
-
+<h2 id="bigqueryio-deadletter-pattern">BigQueryIO deadletter pattern</h2>
+<p>In production systems, it is useful to implement the deadletter pattern with BigQueryIO outputting any elements which had errors during processing by BigQueryIO into another PCollection for further processing.
+The samples below print the errors, but in a production system they can be sent to a deadletter table for later correction.</p>
- <p class="language-py">To read filenames in a pipeline job:</p>
+ <p class="language-java">When using <code>STREAMING_INSERTS</code> you can use the <code>WriteResult</code> object to access a <code>PCollection</code> with the <code>TableRows</code> that failed to be inserted into BigQuery.
+If you also set the <code>withExtendedErrorInfo</code> property , you will be able to access a <code>PCollection<BigQueryInsertError></code> from the <code>WriteResult</code>. The <code>PCollection</code> will then include a reference to the table, the data row and the <code>InsertErrors</code>. Which errors are added to the deadletter queue is determined via the <code>InsertRetryPolicy</code>.</p>
- <p class="language-py"><ol>
-<li>Collect the list of file URIs. You can use the <a href="https://beam.apache.org/releases/pydoc/current/apache_beam.io.filesystems.html?highlight=filesystems#module-apache_beam.io.filesystems"><code>FileSystems</code></a> module to get a list of files that match a glob pattern.</li>
-<li>Pass the file URIs to a <code>PCollection</code>.</li>
-</ol></p>
+ <p class="language-py">In the result tuple you can access <code>FailedRows</code> to access the failed inserts.</p>
@@ -610,19 +514,54 @@ limitations under the License.
<div class=language-java>
- <div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"><span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">FileIO</span><span class="o">.</span><span class="na">match</span><span class="o">().</span><span class="na">filepattern</span><span class="o">(</span><span class="s">"hdfs://path/to/*.gz"</span><span class="o">))</span>
- <span class="c1">// The withCompression method is optional. By default, the Beam SDK detects compression from
-</span><span class="c1"></span> <span class="c1">// the filename.
-</span><span class="c1"></span> <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">FileIO</span><span class="o">.</span><span class="na">readMatches</span><span class="o">().</span><span class="na">withCompression</span><span class="o">(</span><span class="n">Compression</span><span class="o">.</span><span class="na">GZIP</span><span class="o">))</span>
- <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
- <span class="n">ParDo</span><span class="o">.</span><span class="na">of</span><span class="o">(</span>
- <span class="k">new</span> <span class="n">DoFn</span><span class="o"><</span><span class="n">FileIO</span><span class="o">.</span><span class="na">ReadableFile</span><span class="o">,</span> <span class="n">String</span><span class="o">>()</span> <span class="o">{</span>
- <span class="nd">@ProcessElement</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">process</span><span class="o">(</span><span class="nd">@Element</span> <span class="n">FileIO</span><span class="o">.</span><span class="na">ReadableFile</span> <span class="n">file</span><span class="o">)</span> <span class="o">{</span>
- <span class="c1">// We can now access the file and its metadata.
-</span><span class="c1"></span> <span class="n">LOG</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">"File Metadata resourceId is {} "</span><span class="o">,</span> <span class="n">file</span><span class="o">.</span><span class="na">getMetadata</span><span class="o">().</span><span class="na">resourceId</span><span class="o">());</span>
- <span class="o">}</span>
- <span class="o">}));</span></code></pre></div>
+ <div class="highlight"><pre class="chroma"><code class="language-java" data-lang="java"> <span class="n">PipelineOptions</span> <span class="n">options</span> <span class="o">=</span>
+ <span class="n">PipelineOptionsFactory</span><span class="o">.</span><span class="na">fromArgs</span><span class="o">(</span><span class="n">args</span><span class="o">).</span><span class="na">withValidation</span><span class="o">().</span><span class="na">as</span><span class="o">(</span><span class="n">BigQueryOptions</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
+
+ <span class="n">Pipeline</span> <span class="n">p</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="o">.</span><span class="na">create</span><span class="o">(</span><span class="n">options</span><span class="o">);</span>
+
+ <span class="c1">// Create a bug by writing the 2nd value as null. The API will correctly
+</span><span class="c1"></span> <span class="c1">// throw an error when trying to insert a null value into a REQUIRED field.
+</span><span class="c1"></span> <span class="n">WriteResult</span> <span class="n">result</span> <span class="o">=</span>
+ <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Create</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">1</span><span class="o">,</span> <span class="n">2</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">BigQueryIO</span><span class="o">.<</span><span class="n">Integer</span><span class="o">></span><span class="n">write</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withSchema</span><span class="o">(</span>
+ <span class="k">new</span> <span class="n">TableSchema</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">setFields</span><span class="o">(</span>
+ <span class="n">ImmutableList</span><span class="o">.</span><span class="na">of</span><span class="o">(</span>
+ <span class="k">new</span> <span class="n">TableFieldSchema</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">setName</span><span class="o">(</span><span class="s">"num"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">setType</span><span class="o">(</span><span class="s">"INTEGER"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">setMode</span><span class="o">(</span><span class="s">"REQUIRED"</span><span class="o">))))</span>
+ <span class="o">.</span><span class="na">to</span><span class="o">(</span><span class="s">"Test.dummyTable"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withFormatFunction</span><span class="o">(</span><span class="n">x</span> <span class="o">-></span> <span class="k">new</span> <span class="n">TableRow</span><span class="o">().</span><span class="na">set</span><span class="o">(</span><span class="s">"num"</span><span class="o">,</span> <span class="o">(</span><span class="n">x</span> <span class="o">==</span> <span class="n">2</span><span class="o">)</span> <span [...]
+ <span class="o">.</span><span class="na">withFailedInsertRetryPolicy</span><span class="o">(</span><span class="n">InsertRetryPolicy</span><span class="o">.</span><span class="na">retryTransientErrors</span><span class="o">())</span>
+ <span class="c1">// Forcing the bounded pipeline to use streaming inserts
+</span><span class="c1"></span> <span class="o">.</span><span class="na">withMethod</span><span class="o">(</span><span class="n">BigQueryIO</span><span class="o">.</span><span class="na">Write</span><span class="o">.</span><span class="na">Method</span><span class="o">.</span><span class="na">STREAMING_INSERTS</span><span class="o">)</span>
+ <span class="c1">// set the withExtendedErrorInfo property.
+</span><span class="c1"></span> <span class="o">.</span><span class="na">withExtendedErrorInfo</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withCreateDisposition</span><span class="o">(</span><span class="n">BigQueryIO</span><span class="o">.</span><span class="na">Write</span><span class="o">.</span><span class="na">CreateDisposition</span><span class="o">.</span><span class="na">CREATE_IF_NEEDED</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withWriteDisposition</span><span class="o">(</span><span class="n">BigQueryIO</span><span class="o">.</span><span class="na">Write</span><span class="o">.</span><span class="na">WriteDisposition</span><span class="o">.</span><span class="na">WRITE_APPEND</span><span class="o">));</span>
+
+ <span class="n">result</span>
+ <span class="o">.</span><span class="na">getFailedInsertsWithErr</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">apply</span><span class="o">(</span>
+ <span class="n">MapElements</span><span class="o">.</span><span class="na">into</span><span class="o">(</span><span class="n">TypeDescriptors</span><span class="o">.</span><span class="na">strings</span><span class="o">())</span>
+ <span class="o">.</span><span class="na">via</span><span class="o">(</span>
+ <span class="n">x</span> <span class="o">-></span> <span class="o">{</span>
+ <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">" The table was "</span> <span class="o">+</span> <span class="n">x</span><span class="o">.</span><span class="na">getTable</span><span class="o">());</span>
+ <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">" The row was "</span> <span class="o">+</span> <span class="n">x</span><span class="o">.</span><span class="na">getRow</span><span class="o">());</span>
+ <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">" The error was "</span> <span class="o">+</span> <span class="n">x</span><span class="o">.</span><span class="na">getError</span><span class="o">());</span>
+ <span class="k">return</span> <span class="s">""</span><span class="o">;</span>
+ <span class="o">}));</span>
+ <span class="n">p</span><span class="o">.</span><span class="na">run</span><span class="o">();</span>
+
+ <span class="cm">/* Sample Output From the pipeline:
+</span><span class="cm"> <p>The table was GenericData{classInfo=[datasetId, projectId, tableId], {datasetId=Test,projectId=<>, tableId=dummyTable}}
+</span><span class="cm"> <p>The row was GenericData{classInfo=[f], {num=null}}
+</span><span class="cm"> <p>The error was GenericData{classInfo=[errors, index],{errors=[GenericData{classInfo=[debugInfo, location, message, reason], {debugInfo=,location=, message=Missing required field: Msg_0_CLOUD_QUERY_TABLE.num., reason=invalid}}],index=0}}
+</span><span class="cm"> */</span>
+ <span class="o">}</span></code></pre></div>
</div>
@@ -634,15 +573,25 @@ limitations under the License.
<div class=language-py>
- <div class="highlight"><pre class="chroma"><code class="language-py" data-lang="py"><span class="k">with</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">()</span> <span class="k">as</span> <span class="n">p</span><span class="p">:</span>
- <span class="n">readable_files</span> <span class="o">=</span> <span class="p">(</span>
- <span class="n">p</span>
- <span class="o">|</span> <span class="n">fileio</span><span class="o">.</span><span class="n">MatchFiles</span><span class="p">(</span><span class="s1">'hdfs://path/to/*.txt'</span><span class="p">)</span>
- <span class="o">|</span> <span class="n">fileio</span><span class="o">.</span><span class="n">ReadMatches</span><span class="p">()</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Reshuffle</span><span class="p">())</span>
- <span class="n">files_and_contents</span> <span class="o">=</span> <span class="p">(</span>
- <span class="n">readable_files</span>
- <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">read_utf8</span><span class="p">())))< [...]
+ <div class="highlight"><pre class="chroma"><code class="language-py" data-lang="py"> <span class="c1"># Create pipeline.</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="p">({</span><span class="s1">'fields'</span><span class="p">:</span> <span class="p">[{</span><span class="s1">'name'</span><span class="p">:</span> <span class="s1">'a'</span><span class="p">,</span> <span class="s1">'type'</span><span class="p">:</span> <span class="s1">'STRING'</span><span class="p">,</span> <span class="s1">'mode'</span><span class="p">:</span> <span [...]
+
+ <span class="n">p</span> <span class="o">=</span> <span class="n">beam</span><span class="o">.</span><span class="n">Pipeline</span><span class="p">()</span>
+
+ <span class="n">errors</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">p</span> <span class="o">|</span> <span class="s1">'Data'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">Create</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span>
+ <span class="o">|</span> <span class="s1">'CreateBrokenData'</span> <span class="o">>></span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">Map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">src</span><span class="p">:</span> <span class="p">{</span><span class="s1">'a'</span><span class="p">:</span> <span class="n">src</span><span class="p">}</span> <span class="k">if</span> <span class="n">src</span> <span class="o">==</span> <span class="mi">2</span> <span class="k">else</span> <span class="p">{</span><span class= [...]
+ <span class="o">|</span> <span class="s1">'WriteToBigQuery'</span> <span class="o">>></span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">WriteToBigQuery</span><span class="p">(</span>
+ <span class="s2">"<Your Project:Test.dummy_a_table"</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span>
+ <span class="n">insert_retry_strategy</span><span class="o">=</span><span class="s1">'RETRY_ON_TRANSIENT_ERROR'</span><span class="p">,</span>
+ <span class="n">create_disposition</span><span class="o">=</span><span class="s1">'CREATE_IF_NEEDED'</span><span class="p">,</span>
+ <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_APPEND'</span><span class="p">))</span>
+ <span class="n">result</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">errors</span><span class="p">[</span><span class="s1">'FailedRows'</span><span class="p">]</span>
+ <span class="o">|</span> <span class="s1">'PrintErrors'</span> <span class="o">>></span>
+ <span class="n">beam</span><span class="o">.</span><span class="n">FlatMap</span><span class="p">(</span><span class="k">lambda</span> <span class="n">err</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="s2">"Error Found {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">err</span><span class="p">))))</span></code></pre></div>
</div>
diff --git a/website/generated-content/documentation/patterns/custom-io/index.html b/website/generated-content/documentation/patterns/custom-io/index.html
index 397881b..6c5f17f 100644
--- a/website/generated-content/documentation/patterns/custom-io/index.html
+++ b/website/generated-content/documentation/patterns/custom-io/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/patterns/custom-windows/index.html b/website/generated-content/documentation/patterns/custom-windows/index.html
index 927335d..8d0e83b 100644
--- a/website/generated-content/documentation/patterns/custom-windows/index.html
+++ b/website/generated-content/documentation/patterns/custom-windows/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/patterns/file-processing/index.html b/website/generated-content/documentation/patterns/file-processing/index.html
index 37d691f..3037e1b 100644
--- a/website/generated-content/documentation/patterns/file-processing/index.html
+++ b/website/generated-content/documentation/patterns/file-processing/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/patterns/overview/index.html b/website/generated-content/documentation/patterns/overview/index.html
index 345a078..1eeaf9b 100644
--- a/website/generated-content/documentation/patterns/overview/index.html
+++ b/website/generated-content/documentation/patterns/overview/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
@@ -482,6 +495,10 @@ limitations under the License.
<ul>
<li><a href="/documentation/patterns/custom-windows/#using-data-to-dynamically-set-session-window-gaps">Using data to dynamically set session window gaps</a></li>
</ul>
+<p><strong>BigQuery patterns</strong> - Patterns for BigQueryIO</p>
+<ul>
+<li><a href="/documentation/patterns/bigqueryio/#google-bigquery-patterns">Google BigQuery patterns</a></li>
+</ul>
<h2 id="contributing-a-pattern">Contributing a pattern</h2>
<p>To contribute a new pipeline pattern, create an issue with the <a href="https://issues.apache.org/jira/browse/BEAM-7449?jql=labels%20%3D%20pipeline-patterns"><code>pipeline-patterns</code> label</a> and add details to the issue description. See <a href="/contribute/">Get started contributing</a> for more information.</p>
<h2 id="whats-next">What’s next</h2>
diff --git a/website/generated-content/documentation/patterns/pipeline-options/index.html b/website/generated-content/documentation/patterns/pipeline-options/index.html
index 8437265..872df18 100644
--- a/website/generated-content/documentation/patterns/pipeline-options/index.html
+++ b/website/generated-content/documentation/patterns/pipeline-options/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/patterns/side-inputs/index.html b/website/generated-content/documentation/patterns/side-inputs/index.html
index 5f1782a..170dcce 100644
--- a/website/generated-content/documentation/patterns/side-inputs/index.html
+++ b/website/generated-content/documentation/patterns/side-inputs/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/pipelines/create-your-pipeline/index.html b/website/generated-content/documentation/pipelines/create-your-pipeline/index.html
index 5d1295d..9451d15 100644
--- a/website/generated-content/documentation/pipelines/create-your-pipeline/index.html
+++ b/website/generated-content/documentation/pipelines/create-your-pipeline/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/pipelines/design-your-pipeline/index.html b/website/generated-content/documentation/pipelines/design-your-pipeline/index.html
index 9a0f47b..406f212 100644
--- a/website/generated-content/documentation/pipelines/design-your-pipeline/index.html
+++ b/website/generated-content/documentation/pipelines/design-your-pipeline/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/pipelines/test-your-pipeline/index.html b/website/generated-content/documentation/pipelines/test-your-pipeline/index.html
index 1b283b8..8a3838e 100644
--- a/website/generated-content/documentation/pipelines/test-your-pipeline/index.html
+++ b/website/generated-content/documentation/pipelines/test-your-pipeline/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/programming-guide/index.html b/website/generated-content/documentation/programming-guide/index.html
index dfa54b3..056e55b 100644
--- a/website/generated-content/documentation/programming-guide/index.html
+++ b/website/generated-content/documentation/programming-guide/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/resources/learning-resources/index.html b/website/generated-content/documentation/resources/learning-resources/index.html
index f80d6fc..c704c54 100644
--- a/website/generated-content/documentation/resources/learning-resources/index.html
+++ b/website/generated-content/documentation/resources/learning-resources/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/resources/videos-and-podcasts/index.html b/website/generated-content/documentation/resources/videos-and-podcasts/index.html
index f0fd591..26e1464 100644
--- a/website/generated-content/documentation/resources/videos-and-podcasts/index.html
+++ b/website/generated-content/documentation/resources/videos-and-podcasts/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/runners/jstorm/index.html b/website/generated-content/documentation/runners/jstorm/index.html
index 6d54ad4..1418ac0 100644
--- a/website/generated-content/documentation/runners/jstorm/index.html
+++ b/website/generated-content/documentation/runners/jstorm/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/runtime/environments/index.html b/website/generated-content/documentation/runtime/environments/index.html
index e6f0f63..01b4f1e 100644
--- a/website/generated-content/documentation/runtime/environments/index.html
+++ b/website/generated-content/documentation/runtime/environments/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/runtime/model/index.html b/website/generated-content/documentation/runtime/model/index.html
index 62b6982..3bc9606 100644
--- a/website/generated-content/documentation/runtime/model/index.html
+++ b/website/generated-content/documentation/runtime/model/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/runtime/sdk-harness-config/index.html b/website/generated-content/documentation/runtime/sdk-harness-config/index.html
index 25fae57..db6ee85 100644
--- a/website/generated-content/documentation/runtime/sdk-harness-config/index.html
+++ b/website/generated-content/documentation/runtime/sdk-harness-config/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/approximatequantiles/index.html b/website/generated-content/documentation/transforms/java/aggregation/approximatequantiles/index.html
index 6aac537..6c597c2 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/approximatequantiles/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/approximatequantiles/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/approximateunique/index.html b/website/generated-content/documentation/transforms/java/aggregation/approximateunique/index.html
index 583d5a8..2fbe61c 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/approximateunique/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/approximateunique/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/cogroupbykey/index.html b/website/generated-content/documentation/transforms/java/aggregation/cogroupbykey/index.html
index 7e56080..6bd7189 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/cogroupbykey/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/cogroupbykey/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/combine/index.html b/website/generated-content/documentation/transforms/java/aggregation/combine/index.html
index 6ca5539b3..440f845 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/combine/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/combine/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/combinewithcontext/index.html b/website/generated-content/documentation/transforms/java/aggregation/combinewithcontext/index.html
index 06b5024..9514dae 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/combinewithcontext/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/combinewithcontext/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/count/index.html b/website/generated-content/documentation/transforms/java/aggregation/count/index.html
index 82660db..8e1d301 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/count/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/count/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/distinct/index.html b/website/generated-content/documentation/transforms/java/aggregation/distinct/index.html
index 7da8313..e2c70d2 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/distinct/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/distinct/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/groupbykey/index.html b/website/generated-content/documentation/transforms/java/aggregation/groupbykey/index.html
index ff139f6..d238e62 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/groupbykey/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/groupbykey/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/groupintobatches/index.html b/website/generated-content/documentation/transforms/java/aggregation/groupintobatches/index.html
index bd375cf..c0c768e 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/groupintobatches/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/groupintobatches/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/hllcount/index.html b/website/generated-content/documentation/transforms/java/aggregation/hllcount/index.html
index 3becff6..41433aa 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/hllcount/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/hllcount/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/latest/index.html b/website/generated-content/documentation/transforms/java/aggregation/latest/index.html
index 49247dc..d56bb1b 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/latest/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/latest/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/max/index.html b/website/generated-content/documentation/transforms/java/aggregation/max/index.html
index 3f4e9ee..f3bf424 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/max/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/max/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/mean/index.html b/website/generated-content/documentation/transforms/java/aggregation/mean/index.html
index fce6c05..ba1aa7b 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/mean/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/mean/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/min/index.html b/website/generated-content/documentation/transforms/java/aggregation/min/index.html
index d49434c..74ef999 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/min/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/min/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/sample/index.html b/website/generated-content/documentation/transforms/java/aggregation/sample/index.html
index e99c391..08f9d6c 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/sample/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/sample/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/sum/index.html b/website/generated-content/documentation/transforms/java/aggregation/sum/index.html
index 8764530..63240fe 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/sum/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/sum/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/aggregation/top/index.html b/website/generated-content/documentation/transforms/java/aggregation/top/index.html
index 99c5c35..868af0b 100644
--- a/website/generated-content/documentation/transforms/java/aggregation/top/index.html
+++ b/website/generated-content/documentation/transforms/java/aggregation/top/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/filter/index.html b/website/generated-content/documentation/transforms/java/elementwise/filter/index.html
index a5bb27c..f3bd703 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/filter/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/filter/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/flatmapelements/index.html b/website/generated-content/documentation/transforms/java/elementwise/flatmapelements/index.html
index ff25887..a179855 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/flatmapelements/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/flatmapelements/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/keys/index.html b/website/generated-content/documentation/transforms/java/elementwise/keys/index.html
index 757eb7b..131aca4 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/keys/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/keys/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/kvswap/index.html b/website/generated-content/documentation/transforms/java/elementwise/kvswap/index.html
index 3b7b7e6..38da7e3 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/kvswap/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/kvswap/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/mapelements/index.html b/website/generated-content/documentation/transforms/java/elementwise/mapelements/index.html
index db777ea..4bb0d0f 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/mapelements/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/mapelements/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/pardo/index.html b/website/generated-content/documentation/transforms/java/elementwise/pardo/index.html
index b319c73..328856c 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/pardo/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/pardo/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/partition/index.html b/website/generated-content/documentation/transforms/java/elementwise/partition/index.html
index 07b15d2..87e6a30 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/partition/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/partition/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/regex/index.html b/website/generated-content/documentation/transforms/java/elementwise/regex/index.html
index 7a94ef3..72266b9 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/regex/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/regex/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/reify/index.html b/website/generated-content/documentation/transforms/java/elementwise/reify/index.html
index 631e7f3..ca4aee8 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/reify/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/reify/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/tostring/index.html b/website/generated-content/documentation/transforms/java/elementwise/tostring/index.html
index ea6bdbd..e0ccb10 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/tostring/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/tostring/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/values/index.html b/website/generated-content/documentation/transforms/java/elementwise/values/index.html
index 24483c5..904b409 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/values/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/values/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/withkeys/index.html b/website/generated-content/documentation/transforms/java/elementwise/withkeys/index.html
index 56525e9..639ebd6 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/withkeys/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/withkeys/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/elementwise/withtimestamps/index.html b/website/generated-content/documentation/transforms/java/elementwise/withtimestamps/index.html
index 57c6cbe..119c1aa 100644
--- a/website/generated-content/documentation/transforms/java/elementwise/withtimestamps/index.html
+++ b/website/generated-content/documentation/transforms/java/elementwise/withtimestamps/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/other/create/index.html b/website/generated-content/documentation/transforms/java/other/create/index.html
index f1f7f0b..a113be8 100644
--- a/website/generated-content/documentation/transforms/java/other/create/index.html
+++ b/website/generated-content/documentation/transforms/java/other/create/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/other/flatten/index.html b/website/generated-content/documentation/transforms/java/other/flatten/index.html
index 27e7080..8e54015 100644
--- a/website/generated-content/documentation/transforms/java/other/flatten/index.html
+++ b/website/generated-content/documentation/transforms/java/other/flatten/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/other/passert/index.html b/website/generated-content/documentation/transforms/java/other/passert/index.html
index 16e4371..84e801a 100644
--- a/website/generated-content/documentation/transforms/java/other/passert/index.html
+++ b/website/generated-content/documentation/transforms/java/other/passert/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/other/view/index.html b/website/generated-content/documentation/transforms/java/other/view/index.html
index 11e6eb9..96bae00 100644
--- a/website/generated-content/documentation/transforms/java/other/view/index.html
+++ b/website/generated-content/documentation/transforms/java/other/view/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/other/window/index.html b/website/generated-content/documentation/transforms/java/other/window/index.html
index 4632001..f0fc6f9 100644
--- a/website/generated-content/documentation/transforms/java/other/window/index.html
+++ b/website/generated-content/documentation/transforms/java/other/window/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/java/overview/index.html b/website/generated-content/documentation/transforms/java/overview/index.html
index 3e3af16..aa13f81 100644
--- a/website/generated-content/documentation/transforms/java/overview/index.html
+++ b/website/generated-content/documentation/transforms/java/overview/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/approximatequantiles/index.html b/website/generated-content/documentation/transforms/python/aggregation/approximatequantiles/index.html
index 8f345a9..a2135a5 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/approximatequantiles/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/approximatequantiles/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/approximateunique/index.html b/website/generated-content/documentation/transforms/python/aggregation/approximateunique/index.html
index d4be373..2c0427b 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/approximateunique/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/approximateunique/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/cogroupbykey/index.html b/website/generated-content/documentation/transforms/python/aggregation/cogroupbykey/index.html
index 13073e2..b78bb73 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/cogroupbykey/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/cogroupbykey/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/combineglobally/index.html b/website/generated-content/documentation/transforms/python/aggregation/combineglobally/index.html
index c92754a..2ffdff0 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/combineglobally/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/combineglobally/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/combinewithcontext/index.html b/website/generated-content/documentation/transforms/python/aggregation/combinewithcontext/index.html
index 2ec9b43..33371f0 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/combinewithcontext/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/combinewithcontext/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/count/index.html b/website/generated-content/documentation/transforms/python/aggregation/count/index.html
index a0e966d..5a5e10a 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/count/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/count/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/distinct/index.html b/website/generated-content/documentation/transforms/python/aggregation/distinct/index.html
index c41b07a..561ba45 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/distinct/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/distinct/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/groupbykey/index.html b/website/generated-content/documentation/transforms/python/aggregation/groupbykey/index.html
index a4933a3..2da726a 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/groupbykey/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/groupbykey/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/groupintobatches/index.html b/website/generated-content/documentation/transforms/python/aggregation/groupintobatches/index.html
index c2cba7c..240ecfd 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/groupintobatches/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/groupintobatches/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/latest/index.html b/website/generated-content/documentation/transforms/python/aggregation/latest/index.html
index 649d626..e9311af 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/latest/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/latest/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/max/index.html b/website/generated-content/documentation/transforms/python/aggregation/max/index.html
index 097c267..262ae7f 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/max/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/max/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/mean/index.html b/website/generated-content/documentation/transforms/python/aggregation/mean/index.html
index 16a6862..768f88a 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/mean/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/mean/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/min/index.html b/website/generated-content/documentation/transforms/python/aggregation/min/index.html
index e119b64..dc87994 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/min/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/min/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/sample/index.html b/website/generated-content/documentation/transforms/python/aggregation/sample/index.html
index 2e0f2bd..032e69b 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/sample/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/sample/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/sum/index.html b/website/generated-content/documentation/transforms/python/aggregation/sum/index.html
index 34dda59..d8cf20d 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/sum/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/sum/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/aggregation/top/index.html b/website/generated-content/documentation/transforms/python/aggregation/top/index.html
index 2b29446..b08f946 100644
--- a/website/generated-content/documentation/transforms/python/aggregation/top/index.html
+++ b/website/generated-content/documentation/transforms/python/aggregation/top/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/filter/index.html b/website/generated-content/documentation/transforms/python/elementwise/filter/index.html
index fe7457d..eeaf80c 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/filter/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/filter/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html b/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html
index 1ce33e0..d68ea78 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/flatmap/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/keys/index.html b/website/generated-content/documentation/transforms/python/elementwise/keys/index.html
index cdc47b0..49335fd 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/keys/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/keys/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html b/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html
index 0895bfb..310e9f3 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/kvswap/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/map/index.html b/website/generated-content/documentation/transforms/python/elementwise/map/index.html
index 44fe656..a293156 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/map/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/map/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html b/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html
index a52e256..e6aa469 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/pardo/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html b/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
index 9342add..5c792c6 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/partition/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/regex/index.html b/website/generated-content/documentation/transforms/python/elementwise/regex/index.html
index 9afaa5d..ce72be0 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/regex/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/regex/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/reify/index.html b/website/generated-content/documentation/transforms/python/elementwise/reify/index.html
index c3beca5..82a2d02 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/reify/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/reify/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html b/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html
index cdfb67f..ffad2b1 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/tostring/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/values/index.html b/website/generated-content/documentation/transforms/python/elementwise/values/index.html
index af557a3..706764e 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/values/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/values/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/withkeys/index.html b/website/generated-content/documentation/transforms/python/elementwise/withkeys/index.html
index 5a41ba4..2945efd 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/withkeys/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/withkeys/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/elementwise/withtimestamps/index.html b/website/generated-content/documentation/transforms/python/elementwise/withtimestamps/index.html
index ab5d680..5bada4b 100644
--- a/website/generated-content/documentation/transforms/python/elementwise/withtimestamps/index.html
+++ b/website/generated-content/documentation/transforms/python/elementwise/withtimestamps/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/other/create/index.html b/website/generated-content/documentation/transforms/python/other/create/index.html
index 2308294..7c9510d 100644
--- a/website/generated-content/documentation/transforms/python/other/create/index.html
+++ b/website/generated-content/documentation/transforms/python/other/create/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/other/flatten/index.html b/website/generated-content/documentation/transforms/python/other/flatten/index.html
index 10d60b1..5446f18 100644
--- a/website/generated-content/documentation/transforms/python/other/flatten/index.html
+++ b/website/generated-content/documentation/transforms/python/other/flatten/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/other/passert/index.html b/website/generated-content/documentation/transforms/python/other/passert/index.html
index 90ab031..b99cf9c 100644
--- a/website/generated-content/documentation/transforms/python/other/passert/index.html
+++ b/website/generated-content/documentation/transforms/python/other/passert/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/other/reshuffle/index.html b/website/generated-content/documentation/transforms/python/other/reshuffle/index.html
index 98db8fa..c900f57 100644
--- a/website/generated-content/documentation/transforms/python/other/reshuffle/index.html
+++ b/website/generated-content/documentation/transforms/python/other/reshuffle/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/other/view/index.html b/website/generated-content/documentation/transforms/python/other/view/index.html
index 0f78304..39c1bde 100644
--- a/website/generated-content/documentation/transforms/python/other/view/index.html
+++ b/website/generated-content/documentation/transforms/python/other/view/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/other/windowinto/index.html b/website/generated-content/documentation/transforms/python/other/windowinto/index.html
index 0025ea8..ba67d98 100644
--- a/website/generated-content/documentation/transforms/python/other/windowinto/index.html
+++ b/website/generated-content/documentation/transforms/python/other/windowinto/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/documentation/transforms/python/overview/index.html b/website/generated-content/documentation/transforms/python/overview/index.html
index c87073f..1b81f9c 100644
--- a/website/generated-content/documentation/transforms/python/overview/index.html
+++ b/website/generated-content/documentation/transforms/python/overview/index.html
@@ -193,6 +193,18 @@
<li><a href="/documentation/io/built-in/">Built-in I/O connectors</a></li>
<li class="section-nav-item--collapsible">
+ <span class="section-nav-list-title">Built-in I/O connector guides</span>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/io/built-in/parquet/">Apache Parquet I/O connector
+ </a></li>
+ <li><a href="/documentation/io/built-in/hadoop/">Hadoop Input/Output Format IO</a></li>
+ <li><a href="/documentation/io/built-in/hcatalog/">HCatalog IO</a></li>
+ <li><a href="/documentation/io/built-in/google-bigquery/">Google BigQuery I/O connector</a></li>
+ </ul>
+ </li>
+
+
+ <li class="section-nav-item--collapsible">
<span class="section-nav-list-title">Developing new I/O connectors</span>
<ul class="section-nav-list">
@@ -402,6 +414,7 @@
<li><a href="/documentation/patterns/pipeline-options/">Pipeline options</a></li>
<li><a href="/documentation/patterns/custom-io/">Custom I/O</a></li>
<li><a href="/documentation/patterns/custom-windows/">Custom windows</a></li>
+ <li><a href="/documentation/patterns/bigqueryio/">BigQueryIO</a></li>
</ul>
</li>
diff --git a/website/generated-content/sitemap.xml b/website/generated-content/sitemap.xml
index 2f13c6c..2efc937 100644
--- a/website/generated-content/sitemap.xml
+++ b/website/generated-content/sitemap.xml
@@ -668,8 +668,13 @@
</url>
<url>
+ <loc>/documentation/patterns/bigqueryio/</loc>
+ <lastmod>2020-05-26T17:23:44+08:00</lastmod>
+ </url>
+
+ <url>
<loc>/documentation/io/built-in/</loc>
- <lastmod>2020-05-15T00:39:54+02:00</lastmod>
+ <lastmod>2020-05-22T22:16:32-04:00</lastmod>
</url>
<url>
@@ -1049,7 +1054,7 @@
<url>
<loc>/documentation/patterns/overview/</loc>
- <lastmod>2020-05-15T00:39:54+02:00</lastmod>
+ <lastmod>2020-05-26T17:23:44+08:00</lastmod>
</url>
<url>