You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/11/03 19:04:27 UTC
[beam-site] 02/03: Regenerate website content
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/beam-site.git
commit 756b59a73a240edf078e3a1d1733fcc841b40c13
Author: Kenneth Knowles <kl...@google.com>
AuthorDate: Fri Nov 3 12:03:08 2017 -0700
Regenerate website content
---
content/blog/index.html | 117 ++-------
content/coming-soon.html | 117 ++-------
content/contribute/contribution-guide/index.html | 217 ++++++++-------
content/contribute/design-principles/index.html | 200 +++++++-------
content/contribute/docker-images/index.html | 197 +++++++-------
content/contribute/index.html | 174 ++++++------
content/contribute/logos/index.html | 183 +++++++------
content/contribute/maturity-model/index.html | 252 +++++++++---------
.../contribute/presentation-materials/index.html | 179 ++++++-------
.../contribute/ptransform-style-guide/index.html | 199 +++++++-------
content/contribute/release-guide/index.html | 238 ++++++++++-------
content/contribute/runner-guide/index.html | 256 ++++++++++--------
content/contribute/source-repository/index.html | 180 ++++++-------
content/contribute/team/index.html | 177 ++++++-------
content/contribute/testing/index.html | 278 +++++++++++---------
content/contribute/work-in-progress/index.html | 179 ++++++-------
content/css/site.css | 172 +++++++++++-
content/documentation/dsls/sql/index.html | 217 ++++++++-------
content/documentation/execution-model/index.html | 253 +++++++++++-------
content/documentation/index.html | 247 ++++++++++-------
content/documentation/io/authoring-java/index.html | 239 ++++++++++-------
.../documentation/io/authoring-overview/index.html | 250 +++++++++++-------
.../documentation/io/authoring-python/index.html | 236 ++++++++++-------
.../documentation/io/built-in/hadoop/index.html | 245 ++++++++++-------
content/documentation/io/built-in/index.html | 236 ++++++++++-------
content/documentation/io/contributing/index.html | 236 ++++++++++-------
content/documentation/io/io-toc/index.html | 240 ++++++++++-------
content/documentation/io/testing/index.html | 263 ++++++++++++-------
.../pipelines/create-your-pipeline/index.html | 244 ++++++++++-------
.../pipelines/design-your-pipeline/index.html | 249 +++++++++++-------
.../pipelines/test-your-pipeline/index.html | 258 +++++++++++-------
content/documentation/programming-guide/index.html | 292 ++++++++++++++-------
content/documentation/resources/index.html | 260 +++++++++++-------
content/documentation/runners/apex/index.html | 156 +++++------
.../runners/capability-matrix/index.html | 149 ++++-------
content/documentation/runners/dataflow/index.html | 166 +++++-------
content/documentation/runners/direct/index.html | 159 +++++------
content/documentation/runners/flink/index.html | 165 +++++-------
content/documentation/runners/gearpump/index.html | 156 +++++------
content/documentation/runners/jstorm/index.html | 249 +++++++++++-------
content/documentation/runners/mapreduce/index.html | 154 +++++------
content/documentation/runners/spark/index.html | 168 ++++++------
.../sdks/feature-comparison/index.html} | 163 +++++-------
.../documentation/sdks/java-extensions/index.html | 173 ++++++------
content/documentation/sdks/java/index.html | 165 +++++-------
content/documentation/sdks/nexmark/index.html | 279 +++++++++++++-------
.../documentation/sdks/python-custom-io/index.html | 200 +++++++-------
.../sdks/python-pipeline-dependencies/index.html | 176 ++++++-------
.../sdks/python-type-safety/index.html | 183 +++++++------
content/documentation/sdks/python/index.html | 166 +++++-------
content/get-started/beam-overview/index.html | 167 ++++++------
content/get-started/downloads/index.html | 176 ++++++-------
content/get-started/index.html | 161 +++++-------
.../get-started/mobile-gaming-example/index.html | 185 +++++++------
content/get-started/quickstart-java/index.html | 169 ++++++------
content/get-started/quickstart-py/index.html | 179 ++++++-------
content/get-started/support/index.html | 168 ++++++------
content/get-started/wordcount-example/index.html | 193 +++++++-------
content/index.html | 117 ++-------
content/js/fix-menu.js | 64 +++++
content/js/language-switch.js | 13 +
content/js/page-nav.js | 53 ++++
content/js/section-nav.js | 87 ++++++
content/privacy_policy/index.html | 117 ++-------
64 files changed, 6529 insertions(+), 5627 deletions(-)
diff --git a/content/blog/index.html b/content/blog/index.html
index 1f0c6ff..40deb97 100644
--- a/content/blog/index.html
+++ b/content/blog/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/blog/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -28,103 +31,37 @@
<body class="body ">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,7 +77,7 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
<div class="body__contained">
diff --git a/content/coming-soon.html b/content/coming-soon.html
index ebf9ca5..d6576f6 100644
--- a/content/coming-soon.html
+++ b/content/coming-soon.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/coming-soon.html" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -28,103 +31,37 @@
<body class="body ">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,7 +77,7 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
<div class="body__contained">
diff --git a/content/contribute/contribution-guide/index.html b/content/contribute/contribution-guide/index.html
index bf89343..f1ca1ff 100644
--- a/content/contribute/contribution-guide/index.html
+++ b/content/contribute/contribution-guide/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/contribution-guide/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,104 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-contribution-guide">Apache Beam Contribution Guide</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#engage">Engage</a>
+ <ul>
+ <li><a href="#mailing-lists">Mailing list(s)</a></li>
+ <li><a href="#jira-issue-tracker">JIRA issue tracker</a></li>
+ <li><a href="#online-discussions">Online discussions</a></li>
+ </ul>
+ </li>
+ <li><a href="#design">Design</a></li>
+ <li><a href="#code">Code</a>
+ <ul>
+ <li><a href="#one-time-setup">One-time Setup</a></li>
+ <li><a href="#create-a-branch-in-your-fork">Create a branch in your fork</a></li>
+ <li><a href="#syncing-and-pushing-your-branch">Syncing and pushing your branch</a></li>
+ <li><a href="#testing">Testing</a></li>
+ </ul>
+ </li>
+ <li><a href="#review">Review</a>
+ <ul>
+ <li><a href="#create-a-pull-request">Create a pull request</a></li>
+ <li><a href="#code-review-and-revision">Code Review and Revision</a></li>
+ <li><a href="#lgtm">LGTM</a></li>
+ <li><a href="#deleting-your-branch">Deleting your branch</a></li>
+ <li><a href="#stale-pull-requests">Stale pull requests</a></li>
+ </ul>
+ </li>
+ <li><a href="#commit-committers-only">Commit (committers only)</a>
+ <ul>
+ <li><a href="#one-time-setup-1">One-time Setup</a></li>
+ <li><a href="#contributor-license-agreement">Contributor License Agreement</a></li>
+ <li><a href="#tests">Tests</a></li>
+ <li><a href="#finishing-touches">Finishing touches</a></li>
+ <li><a href="#merge-process">Merge process</a></li>
+ </ul>
+ </li>
+ <li><a href="#granting-more-rights-to-a-contributor">Granting more rights to a contributor</a></li>
+ <li><a href="#special-cases">Special Cases</a>
+ <ul>
+ <li><a href="#feature-branches">Feature Branches</a></li>
+ <li><a href="#website">Website</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-contribution-guide">Apache Beam Contribution Guide</h1>
<ul id="markdown-toc">
<li><a href="#engage" id="markdown-toc-engage">Engage</a> <ul>
@@ -805,6 +835,7 @@ $ git commit -m "Regenerate website"
<p>Finally you should merge the changes into the <code class="highlighter-rouge">asf-site</code> branch and push them into the <code class="highlighter-rouge">apache</code> repository.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/design-principles/index.html b/content/contribute/design-principles/index.html
index 9dbd062..7732119 100644
--- a/content/contribute/design-principles/index.html
+++ b/content/contribute/design-principles/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/design-principles/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,86 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="design-principles-in-the-apache-beam-project">Design Principles in the Apache Beam Project</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#use-cases">Use cases</a>
+ <ul>
+ <li><a href="#unify-the-model">Unify the model</a></li>
+ <li><a href="#separate-data-shapes-and-runtime-requirements">Separate data shapes and runtime requirements</a></li>
+ <li><a href="#make-efficient-things-easy-rather-than-make-easy-things-efficient">Make efficient things easy, rather than make easy things efficient</a></li>
+ </ul>
+ </li>
+ <li><a href="#usability">Usability</a>
+ <ul>
+ <li><a href="#validate-early">Validate Early</a></li>
+ <li><a href="#public-apis-like-diamonds-are-forever-at-least-until-the-next-major-version">Public APIs, like diamonds, are forever (at least until the next major version)</a></li>
+ <li><a href="#examples-should-be-pedagogical">Examples should be pedagogical</a></li>
+ </ul>
+ </li>
+ <li><a href="#extensibility">Extensibility</a>
+ <ul>
+ <li><a href="#use-ptransforms-for-modularity">Use PTransforms for modularity</a></li>
+ <li><a href="#keep-beam-sdks-consistent">Keep Beam SDKs consistent</a></li>
+ <li><a href="#when-in-rome-python-do-as-the-romans-pythonians-do">When in <del>Rome</del> Python, do as the <del>Romans</del> Pythonians do</a></li>
+ <li><a href="#encourage-dsls">Encourage DSLs</a></li>
+ <li><a href="#design-for-the-model-not-specific-runners">Design for the model, not specific runners</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="design-principles-in-the-apache-beam-project">Design Principles in the Apache Beam Project</h1>
<p>Joshua Bloch’s <a href="https://www.infoq.com/articles/API-Design-Joshua-Bloch">API Design Bumper Stickers</a> are a great list of what makes for good API design. In addition, we have specific design principles we follow in Beam.</p>
@@ -211,7 +223,7 @@
<p>The Beam APIs should serve all runners. Behind every runner-specific hook, there is a general principle in the model. Design APIs that generalize across multiple runners.</p>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/docker-images/index.html b/content/contribute/docker-images/index.html
index c17cab2..b9d618e 100644
--- a/content/contribute/docker-images/index.html
+++ b/content/contribute/docker-images/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/docker-images/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,72 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="docker-images">Docker Images</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#file-based-image">File based image</a>
+ <ul>
+ <li><a href="#configuring-the-runtime-via-the-environment-variables">Configuring the runtime via the environment variables</a></li>
+ <li><a href="#testing-in-an-specific-environment-with-your-own-source">Testing in an specific environment with your own source</a></li>
+ <li><a href="#performance-improvements">Performance improvements</a></li>
+ </ul>
+ </li>
+ <li><a href="#git-based-image">Git based image</a></li>
+ <li><a href="#release-image">Release image</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="docker-images">Docker Images</h1>
<p>Docker images allow to create a reproducible environment to build and test
Beam. You can use the docker images by using the provided <a href="https://github.com/apache/beam/tree/master/sdks/java/build-tools/src/main/resources/docker">Docker scripts</a>.</p>
@@ -154,23 +152,23 @@ different purposes:</p>
<ul>
<li>
- <p><a href="#file">file</a>: Create a Docker container from a Beam source code .zip file
+ <p><a href="#file-based-image">file</a>: Create a Docker container from a Beam source code .zip file
in a given environment. It is useful to test a specific version of Beam,
for example to validate a release vote.</p>
</li>
<li>
- <p><a href="#git">git</a>: Same as file but the Beam source code comes from the git repository,
+ <p><a href="#git-based-image">git</a>: Same as file but the Beam source code comes from the git repository,
you can choose a given branch/tag/pull-request. Useful to test in a specific
environment.</p>
</li>
<li>
- <p><a href="#release">release</a>: It builds an end-user distribution of the latest version of Beam
+ <p><a href="#release-image">release</a>: It builds an end-user distribution of the latest version of Beam
and its dependencies. Useful for end-users who want to have a ready to use
container with Beam (Python only for the moment).</p>
</li>
</ul>
-<h2 id="a-namefileafile-based-image"><a name="file"></a>File based image</h2>
+<h2 id="file-based-image">File based image</h2>
<p>If you want to build a container with a ready JDK 8 environment to test Beam:</p>
@@ -297,7 +295,7 @@ enable the threading execution:</p>
</code></pre>
</div>
-<h2 id="a-namegitagit-based-image"><a name="git"></a>Git based image</h2>
+<h2 id="git-based-image">Git based image</h2>
<p>It creates a docker container with a cloned version of the git repository, its
branches and all the pull-requests. When it is run, it updates and checks out
@@ -317,7 +315,7 @@ docker run -e BRANCH=pr/ID -it beam:git /bin/bash
</code></pre>
</div>
-<h2 id="a-namereleasearelease-image"><a name="release"></a>Release image</h2>
+<h2 id="release-image">Release image</h2>
<p>The release image is a container with the needed requirements to run Beam out of
the box in a container, or to submit a pipeline to execute on Google Cloud
@@ -328,6 +326,7 @@ docker build -t beam:python2 .
</code></pre>
</div>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/index.html b/content/contribute/index.html
index 3c8288e..4aaa10c 100644
--- a/content/contribute/index.html
+++ b/content/contribute/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,61 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="contribute-to-the-apache-beam-project">Contribute to the Apache Beam Project</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="contribute-to-the-apache-beam-project">Contribute to the Apache Beam Project</h1>
<p>Learn how you can contribute to Beam:</p>
@@ -166,6 +153,7 @@
<li><a href="/get-started/support/">Issue Tracking</a></li>
</ul>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/logos/index.html b/content/contribute/logos/index.html
index 44347b5..339f07f 100644
--- a/content/contribute/logos/index.html
+++ b/content/contribute/logos/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/logos/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,70 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-logos">Apache Beam Logos</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#project-logos">Project logos</a>
+ <ul>
+ <li><a href="#scalable-vector-graphics-svg">Scalable Vector Graphics (SVG)</a></li>
+ <li><a href="#portable-network-graphics-png">Portable Network Graphics (PNG)</a></li>
+ </ul>
+ </li>
+ <li><a href="#colors-and-fonts">Colors and fonts</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-logos">Apache Beam Logos</h1>
<p>This page contains project material for the Apache Beam project.</p>
@@ -449,6 +445,7 @@ available in a number of fixed sizes and are optimized for web use.</p>
<h2 id="colors-and-fonts">Colors and fonts</h2>
<p>The Apache Beam project uses predefined colors and fonts. <a href="/downloads/palette.pdf">This document</a> has more information.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/maturity-model/index.html b/content/contribute/maturity-model/index.html
index bb7f360..9dfed21 100644
--- a/content/contribute/maturity-model/index.html
+++ b/content/contribute/maturity-model/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/maturity-model/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,71 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-maturity-model-assessment-for-apache-beam">Apache Maturity Model Assessment for Apache Beam</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#maturity-model">Maturity model</a></li>
+ <li><a href="#independence--community">Independence / Community</a></li>
+ <li><a href="#dependency-analysis">Dependency analysis</a>
+ <ul>
+ <li><a href="#license-analysis">License analysis</a></li>
+ <li><a href="#dependency-tree">Dependency tree</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-maturity-model-assessment-for-apache-beam">Apache Maturity Model Assessment for Apache Beam</h1>
<p><em>Apache Beam has graduated from incubation as a top-level project at the
Apache Software Foundation. This page was last updated as a part of the
@@ -829,7 +826,7 @@ graduation process and is no longer being maintained.</em></p>
<div class="highlighter-rouge"><pre class="highlight"><code>[INFO] Scanning for projects...
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Build Order:
-[INFO]
+[INFO]
[INFO] Apache Beam :: Parent
[INFO] Apache Beam :: SDKs :: Java :: Build Tools
[INFO] Apache Beam :: SDKs
@@ -866,35 +863,35 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Parent 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-parent ---
[INFO] org.apache.beam:beam-parent:pom:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: Build Tools 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-build-tools ---
[INFO] org.apache.beam:beam-sdks-java-build-tools:jar:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-parent ---
[INFO] org.apache.beam:beam-sdks-parent:pom:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-parent ---
[INFO] org.apache.beam:beam-sdks-java-parent:pom:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: Core 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-core ---
[INFO] org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- io.grpc:grpc-auth:jar:1.0.1:compile
@@ -1066,14 +1063,14 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Runners 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-runners-parent ---
[INFO] org.apache.beam:beam-runners-parent:pom:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Runners :: Core Java 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-runners-core-java ---
[INFO] org.apache.beam:beam-runners-core-java:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -1267,7 +1264,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Runners :: Direct Java 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-runners-direct-java ---
[INFO] org.apache.beam:beam-runners-direct-java:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -1489,7 +1486,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Runners :: Google Cloud Dataflow 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-runners-google-cloud-dataflow-java ---
[INFO] org.apache.beam:beam-runners-google-cloud-dataflow-java:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -1711,14 +1708,14 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: IO 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-io-parent ---
[INFO] org.apache.beam:beam-sdks-java-io-parent:pom:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: IO :: Google Cloud Platform 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-io-google-cloud-platform ---
[INFO] org.apache.beam:beam-sdks-java-io-google-cloud-platform:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -2000,7 +1997,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: IO :: HDFS 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-io-hdfs ---
[INFO] org.apache.beam:beam-sdks-java-io-hdfs:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -2393,7 +2390,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: IO :: JMS 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-io-jms ---
[INFO] org.apache.beam:beam-sdks-java-io-jms:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -2577,7 +2574,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: IO :: Kafka 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-io-kafka ---
[INFO] org.apache.beam:beam-sdks-java-io-kafka:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -2751,7 +2748,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: IO :: Kinesis 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-io-kinesis ---
[INFO] org.apache.beam:beam-sdks-java-io-kinesis:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -2949,7 +2946,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: IO :: MongoDB 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-io-mongodb ---
[INFO] org.apache.beam:beam-sdks-java-io-mongodb:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -3130,7 +3127,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: IO :: JDBC 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-io-jdbc ---
[INFO] org.apache.beam:beam-sdks-java-io-jdbc:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -3306,14 +3303,14 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: Maven Archetypes 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-maven-archetypes-parent ---
[INFO] org.apache.beam:beam-sdks-java-maven-archetypes-parent:pom:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: Maven Archetypes :: Starter 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-maven-archetypes-starter ---
[INFO] org.apache.beam:beam-sdks-java-maven-archetypes-starter:maven-archetype:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:runtime
@@ -3460,7 +3457,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: Maven Archetypes :: Examples 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-maven-archetypes-examples ---
[INFO] org.apache.beam:beam-sdks-java-maven-archetypes-examples:maven-archetype:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:runtime
@@ -3709,14 +3706,14 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: Extensions 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-extensions-parent ---
[INFO] org.apache.beam:beam-sdks-java-extensions-parent:pom:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: Extensions :: Join library 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-extensions-join-library ---
[INFO] org.apache.beam:beam-sdks-java-extensions-join-library:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -3880,7 +3877,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: Extensions :: Sorter 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-extensions-sorter ---
[INFO] org.apache.beam:beam-sdks-java-extensions-sorter:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -4195,7 +4192,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: SDKs :: Java :: Java 8 Tests 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-sdks-java-java8tests ---
[INFO] org.apache.beam:beam-sdks-java-java8tests:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:test
@@ -4360,14 +4357,14 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Runners :: Flink 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-runners-flink-parent ---
[INFO] org.apache.beam:beam-runners-flink-parent:pom:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Runners :: Flink :: Core 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-runners-flink_2.10 ---
[INFO] org.apache.beam:beam-runners-flink_2.10:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.flink:flink-streaming-java_2.10:jar:1.1.2:compile
@@ -4806,7 +4803,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Runners :: Flink :: Examples 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-runners-flink_2.10-examples ---
[WARNING] The parameter output is deprecated. Use outputFile instead.
[INFO] Wrote dependency tree to: /usr/local/google/home/davor/GitHub/incubator-beam/runners/flink/examples/wordcounts.txt
@@ -4814,7 +4811,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Runners :: Spark 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-runners-spark ---
[INFO] org.apache.beam:beam-runners-spark:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.spark:spark-core_2.10:jar:1.6.2:provided
@@ -5398,7 +5395,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Runners :: Apex 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-runners-apex ---
[INFO] org.apache.beam:beam-runners-apex:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.apex:apex-common:jar:3.5.0-SNAPSHOT:compile
@@ -5832,14 +5829,14 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Examples 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-examples-parent ---
[INFO] org.apache.beam:beam-examples-parent:pom:0.4.0-incubating-SNAPSHOT
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Examples :: Java 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-examples-java ---
[INFO] org.apache.beam:beam-examples-java:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -6074,7 +6071,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] ------------------------------------------------------------------------
[INFO] Building Apache Beam :: Examples :: Java 8 0.4.0-incubating-SNAPSHOT
[INFO] ------------------------------------------------------------------------
-[INFO]
+[INFO]
[INFO] --- maven-dependency-plugin:2.10:tree (default-cli) @ beam-examples-java8 ---
[INFO] org.apache.beam:beam-examples-java8:jar:0.4.0-incubating-SNAPSHOT
[INFO] +- org.apache.beam:beam-sdks-java-core:jar:0.4.0-incubating-SNAPSHOT:compile
@@ -6768,7 +6765,7 @@ graduation process and is no longer being maintained.</em></p>
[INFO] \- (org.spark-project.spark:unused:jar:1.0.0:runtime - omitted for duplicate)
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
-[INFO]
+[INFO]
[INFO] Apache Beam :: Parent ............................. SUCCESS [0.821s]
[INFO] Apache Beam :: SDKs :: Java :: Build Tools ........ SUCCESS [0.008s]
[INFO] Apache Beam :: SDKs ............................... SUCCESS [0.007s]
@@ -6811,6 +6808,7 @@ graduation process and is no longer being maintained.</em></p>
</code></pre>
</div>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/presentation-materials/index.html b/content/contribute/presentation-materials/index.html
index 4be107f..8858ef9 100644
--- a/content/contribute/presentation-materials/index.html
+++ b/content/contribute/presentation-materials/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/presentation-materials/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,66 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-presentation-materials">Apache Beam Presentation Materials</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#using-these-materials">Using these materials</a></li>
+ <li><a href="#contributing-new-content">Contributing new content</a></li>
+ <li><a href="#corrections-and-updates">Corrections and updates</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-presentation-materials">Apache Beam Presentation Materials</h1>
<p>Are you interested in learning more about Apache Beam or giving a talk about Apache Beam? Excellent! The Apache Beam community has created this collection of materials to help you get started. The community periodically reviews these materials to ensure they are accurate and up-to-date.</p>
@@ -169,6 +161,7 @@
<h2 id="corrections-and-updates">Corrections and updates</h2>
<p>If you encounter missing, incorrect, or out of date material, please make a comment in the presentation or send an email to the <a href="mailto:user@beam.apache.org"><code class="highlighter-rouge">user@beam.apache.org</code></a> mailing list.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/ptransform-style-guide/index.html b/content/contribute/ptransform-style-guide/index.html
index 5a9d17a..6a2a13d 100644
--- a/content/contribute/ptransform-style-guide/index.html
+++ b/content/contribute/ptransform-style-guide/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/ptransform-style-guide/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,83 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="ptransform-style-guide">PTransform Style Guide</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#language-neutral-considerations">Language-neutral considerations</a>
+ <ul>
+ <li><a href="#consistency">Consistency</a></li>
+ <li><a href="#exposing-a-ptransform-vs-something-else">Exposing a PTransform vs. something else</a></li>
+ <li><a href="#naming">Naming</a></li>
+ <li><a href="#configuration">Configuration</a></li>
+ <li><a href="#error-handling">Error handling</a></li>
+ <li><a href="#performance">Performance</a></li>
+ <li><a href="#documentation">Documentation</a></li>
+ <li><a href="#logging">Logging</a></li>
+ <li><a href="#testing">Testing</a></li>
+ <li><a href="#compatibility">Compatibility</a></li>
+ </ul>
+ </li>
+ <li><a href="#java-specific-considerations">Java specific considerations</a>
+ <ul>
+ <li><a href="#api">API</a></li>
+ <li><a href="#behavior">Behavior</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="ptransform-style-guide">PTransform Style Guide</h1>
<p><em>A style guide for writers of new reusable PTransforms.</em></p>
@@ -591,7 +600,7 @@ Strive to make such incompatible behavior changes cause a compile error (e.g. it
<span class="kd">public</span> <span class="kd">static</span> <span class="n">Twiddle</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="nf">twiddle</span><span class="o">()</span> <span class="o">{</span>
<span class="k">return</span> <span class="k">new</span> <span class="n">AutoValue_Thumbs_Twiddle</span><span class="o">.</span><span class="na">Builder</span><span class="o"><</span><span class="n">T</span><span class="o">>().</span><span class="na">build</span><span class="o">();</span>
<span class="o">}</span>
-
+
<span class="nd">@AutoValue</span>
<span class="kd">public</span> <span class="kd">abstract</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">Twiddle</span><span class="o"><</span><span class="n">T</span><span class="o">></span>
<span class="kd">extends</span> <span class="n">PTransform</span><span class="o"><</span><span class="n">PCollection</span><span class="o"><</span><span class="n">Foo</span><span class="o">>,</span> <span class="n">PCollection</span><span class="o"><</span><span class="n">Bar</span><span class="o"><</span><span class="n">T</span><span class="o">>>></span> <span class="o">{</span>
@@ -798,7 +807,7 @@ Most <code class="highlighter-rouge">PTransform</code>s do not use <code class="
<li>Always make it possible for the user to explicitly specify a <code class="highlighter-rouge">Coder</code> for the relevant type variable(s) as a configuration parameter of your <code class="highlighter-rouge">PTransform</code>. (e.g. <code class="highlighter-rouge">AvroIO.<T>parseGenericRecords().withCoder(Coder<T>)</code>). Fall back to inference if the coder was not explicitly specified.</li>
</ul>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/release-guide/index.html b/content/contribute/release-guide/index.html
index 4c304bf..9844c53 100644
--- a/content/contribute/release-guide/index.html
+++ b/content/contribute/release-guide/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/release-guide/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,123 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-release-guide">Apache Beam Release Guide</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#introduction">Introduction</a></li>
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#decide-to-release">Decide to release</a>
+ <ul>
+ <li><a href="#checklist-to-proceed-to-the-next-step">Checklist to proceed to the next step</a></li>
+ </ul>
+ </li>
+ <li><a href="#prepare-for-the-release">Prepare for the release</a>
+ <ul>
+ <li><a href="#one-time-setup-instructions">One-time setup instructions</a></li>
+ <li><a href="#create-a-new-version-in-jira">Create a new version in JIRA</a></li>
+ <li><a href="#triage-release-blocking-issues-in-jira">Triage release-blocking issues in JIRA</a></li>
+ <li><a href="#review-release-notes-in-jira">Review Release Notes in JIRA</a></li>
+ <li><a href="#verify-that-a-release-build-works">Verify that a Release Build Works</a></li>
+ <li><a href="#update-and-verify-javadoc">Update and Verify Javadoc</a></li>
+ <li><a href="#create-a-release-branch">Create a release branch</a></li>
+ <li><a href="#update-the-python-sdk-version">Update the Python SDK version</a></li>
+ <li><a href="#update-release-specific-configurations">Update release specific configurations</a></li>
+ <li><a href="#checklist-to-proceed-to-the-next-step-1">Checklist to proceed to the next step</a></li>
+ </ul>
+ </li>
+ <li><a href="#build-a-release-candidate">Build a release candidate</a>
+ <ul>
+ <li><a href="#build-and-stage-java-artifacts-with-maven">Build and stage Java artifacts with Maven</a></li>
+ <li><a href="#stage-source-release-on-distapacheorg">Stage source release on dist.apache.org</a></li>
+ <li><a href="#build-the-pydoc-api-reference">Build the Pydoc API reference</a></li>
+ <li><a href="#propose-a-pull-request-for-website-updates">Propose a pull request for website updates</a></li>
+ <li><a href="#checklist-to-proceed-to-the-next-step-2">Checklist to proceed to the next step</a></li>
+ </ul>
+ </li>
+ <li><a href="#vote-on-the-release-candidate">Vote on the release candidate</a>
+ <ul>
+ <li><a href="#checklist-to-proceed-to-the-finalization-step">Checklist to proceed to the finalization step</a></li>
+ </ul>
+ </li>
+ <li><a href="#fix-any-issues">Fix any issues</a>
+ <ul>
+ <li><a href="#checklist-to-proceed-to-the-next-step-3">Checklist to proceed to the next step</a></li>
+ </ul>
+ </li>
+ <li><a href="#finalize-the-release">Finalize the release</a>
+ <ul>
+ <li><a href="#deploy-artifacts-to-maven-central-repository">Deploy artifacts to Maven Central Repository</a></li>
+ <li><a href="#deploy-python-artifacts-to-pypi">Deploy Python artifacts to PyPI</a></li>
+ <li><a href="#git-tag">Git tag</a></li>
+ <li><a href="#merge-website-pull-request">Merge website pull request</a></li>
+ <li><a href="#mark-the-version-as-released-in-jira">Mark the version as released in JIRA</a></li>
+ <li><a href="#checklist-to-proceed-to-the-next-step-4">Checklist to proceed to the next step</a></li>
+ </ul>
+ </li>
+ <li><a href="#promote-the-release">Promote the release</a>
+ <ul>
+ <li><a href="#apache-mailing-lists">Apache mailing lists</a></li>
+ <li><a href="#recordkeeping">Recordkeeping</a></li>
+ <li><a href="#beam-blog">Beam blog</a></li>
+ <li><a href="#social-media">Social media</a></li>
+ <li><a href="#checklist-to-declare-the-process-completed">Checklist to declare the process completed</a></li>
+ </ul>
+ </li>
+ <li><a href="#improve-the-process">Improve the process</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-release-guide">Apache Beam Release Guide</h1>
<ul id="markdown-toc">
<li><a href="#introduction" id="markdown-toc-introduction">Introduction</a></li>
@@ -658,7 +707,7 @@ The complete staging area is available for your review, which includes:
* source code tag "v1.2.3-RC3" [5],
* website pull request listing the release and publishing the API reference manual [6].
* Java artifacts were built with Maven MAVEN_VERSION and OpenJDK/Oracle JDK JDK_VERSION.
-* Python artifacts are deployed along with the source release to the dist.apache.org [2].
+* Python artifacts are deployed along with the source release to the dist.apache.org [2].
The vote will be open for at least 72 hours. It is adopted by majority approval, with at least 3 PMC affirmative votes.
@@ -812,6 +861,7 @@ Thanks everyone!
<p>If we have specific ideas, please start a discussion on the dev@ mailing list and/or propose a pull request to update this guide. Thanks!</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/runner-guide/index.html b/content/contribute/runner-guide/index.html
index e484eaa..1b990ac 100644
--- a/content/contribute/runner-guide/index.html
+++ b/content/contribute/runner-guide/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/runner-guide/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,118 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="runner-authoring-guide">Runner Authoring Guide</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#basics-of-the-beam-model">Basics of the Beam model</a>
+ <ul>
+ <li><a href="#pipeline">Pipeline</a></li>
+ <li><a href="#ptransforms">PTransforms</a></li>
+ <li><a href="#pcollections">PCollections</a></li>
+ <li><a href="#user-defined-functions-udfs">User-Defined Functions (UDFs)</a></li>
+ <li><a href="#runner">Runner</a></li>
+ </ul>
+ </li>
+ <li><a href="#implementing-the-beam-primitives">Implementing the Beam Primitives</a>
+ <ul>
+ <li><a href="#what-if-you-havent-implemented-some-of-these-features">What if you haven’t implemented some of these features?</a></li>
+ <li><a href="#implementing-the-pardo-primitive">Implementing the ParDo primitive</a></li>
+ <li><a href="#implementing-the-groupbykey-and-window-primitive">Implementing the GroupByKey (and window) primitive</a></li>
+ <li><a href="#implementing-the-window-primitive">Implementing the Window primitive</a></li>
+ <li><a href="#implementing-the-read-primitive">Implementing the Read primitive</a></li>
+ <li><a href="#implementing-the-flatten-primitive">Implementing the Flatten primitive</a></li>
+ <li><a href="#special-mention-the-combine-composite">Special mention: the Combine composite</a></li>
+ </ul>
+ </li>
+ <li><a href="#working-with-pipelines">Working with pipelines</a>
+ <ul>
+ <li><a href="#traversing-a-pipeline">Traversing a pipeline</a></li>
+ <li><a href="#altering-a-pipeline">Altering a pipeline</a></li>
+ </ul>
+ </li>
+ <li><a href="#testing-your-runner">Testing your runner</a></li>
+ <li><a href="#integrating-your-runner-nicely-with-sdks">Integrating your runner nicely with SDKs</a>
+ <ul>
+ <li><a href="#integrating-with-the-java-sdk">Integrating with the Java SDK</a></li>
+ <li><a href="#integrating-with-the-python-sdk">Integrating with the Python SDK</a></li>
+ </ul>
+ </li>
+ <li><a href="#writing-an-sdk-independent-runner">Writing an SDK-independent runner</a>
+ <ul>
+ <li><a href="#the-fn-api">The Fn API</a></li>
+ <li><a href="#the-runner-api">The Runner API</a></li>
+ </ul>
+ </li>
+ <li><a href="#the-runner-api-protos">The Runner API protos</a>
+ <ul>
+ <li><a href="#functionspec-proto"><code class="highlighter-rouge">FunctionSpec</code> proto</a></li>
+ <li><a href="#sdkfunctionspec-proto"><code class="highlighter-rouge">SdkFunctionSpec</code> proto</a></li>
+ <li><a href="#primitive-transform-payload-protos">Primitive transform payload protos</a></li>
+ <li><a href="#ptransform-proto"><code class="highlighter-rouge">PTransform</code> proto</a></li>
+ <li><a href="#pcollection-proto"><code class="highlighter-rouge">PCollection</code> proto</a></li>
+ <li><a href="#coder-proto"><code class="highlighter-rouge">Coder</code> proto</a></li>
+ </ul>
+ </li>
+ <li><a href="#the-runner-api-rpcs">The Runner API RPCs</a>
+ <ul>
+ <li><a href="#pipelinerunnerrunpipeline-rpc"><code class="highlighter-rouge">PipelineRunner.run(Pipeline)</code> RPC</a></li>
+ <li><a href="#pipelineresult-aka-job-api"><code class="highlighter-rouge">PipelineResult</code> aka “Job API”</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="runner-authoring-guide">Runner Authoring Guide</h1>
<p>This guide walks through how to implement a new runner. It is aimed at someone
who has a data processing system and wants to use it to execute a Beam
@@ -284,7 +328,7 @@ composite transform encapsulating a subgraph. The primitives are:</p>
<li><a href="#implementing-the-read-primitive"><em>Read</em></a> - parallel connectors to external
systems</li>
<li><a href="#implementing-the-pardo-primitive"><em>ParDo</em></a> - per element processing</li>
- <li><a href="#implementing-the-groupbykey-and-window-primitive"><em>GroupByKey</em></a> -
+ <li><a href="#implementing-the-groupbykey-and-window-primitive"><em>GroupByKey</em></a> -
aggregating elements per key and window</li>
<li><a href="#implementing-the-flatten-primitive"><em>Flatten</em></a> - union of PCollections</li>
<li><a href="#implementing-the-window-primitive"><em>Window</em></a> - set the windowing strategy
@@ -535,15 +579,15 @@ the Python codebase.</p>
<p>There are some implementations and variations of this for different scenarios:</p>
<ul>
- <li><a href="https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java"><code class="highlighter-rouge">SimpleDoFnRunner</code></a> -
+ <li><a href="https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java"><code class="highlighter-rouge">SimpleDoFnRunner</code></a> -
not actually simple at all; implements lots of the core functionality of
<code class="highlighter-rouge">ParDo</code>. This is how most runners execute most <code class="highlighter-rouge">DoFns</code>.</li>
- <li><a href="https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java"><code class="highlighter-rouge">LateDataDroppingDoFnRunner</code></a> -
+ <li><a href="https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/LateDataDroppingDoFnRunner.java"><code class="highlighter-rouge">LateDataDroppingDoFnRunner</code></a> -
wraps a <code class="highlighter-rouge">DoFnRunner</code> and drops data from expired windows so the wrapped
<code class="highlighter-rouge">DoFnRunner</code> doesn’t get any unpleasant surprises</li>
- <li><a href="https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java"><code class="highlighter-rouge">StatefulDoFnRunner</code></a> -
+ <li><a href="https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/StatefulDoFnRunner.java"><code class="highlighter-rouge">StatefulDoFnRunner</code></a> -
handles collecting expired state</li>
- <li><a href="https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java"><code class="highlighter-rouge">PushBackSideInputDoFnRunner</code></a> -
+ <li><a href="https://github.com/apache/beam/blob/master/runners/core-java/src/main/java/org/apache/beam/runners/core/PushbackSideInputDoFnRunner.java"><code class="highlighter-rouge">PushBackSideInputDoFnRunner</code></a> -
buffers input while waiting for side inputs to be ready</li>
</ul>
@@ -890,7 +934,7 @@ scan the dependencies of the SDK for tests with the JUnit category
<span class="nt"><beamTestPipelineOptions></span>
[
"--runner=MyRunner",
- … misc test options …
+ … misc test options …
]
<span class="nt"></beamTestPipelineOptions></span>
<span class="nt"></systemPropertyVariables></span>
@@ -926,7 +970,7 @@ with matching names, like so:</p>
<span class="nd">@Required</span>
<span class="kd">public</span> <span class="n">Foo</span> <span class="nf">getMyRequiredFoo</span><span class="o">();</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">setMyRequiredFoo</span><span class="o">(</span><span class="n">Foo</span> <span class="n">newValue</span><span class="o">);</span>
-
+
<span class="nd">@Description</span><span class="o">(</span><span class="s">"Enable Baz; on by default"</span><span class="o">)</span>
<span class="nd">@Default</span><span class="o">.</span><span class="na">Boolean</span><span class="o">(</span><span class="kc">true</span><span class="o">)</span>
<span class="kd">public</span> <span class="n">Boolean</span> <span class="nf">isBazEnabled</span><span class="o">();</span>
@@ -1211,7 +1255,7 @@ be passed back to an SDK harness; they do not represent a UDF.</p>
<div class="language-proto no-toggle highlighter-rouge"><pre class="highlight"><code><span class="kd">message</span> <span class="nc">PTransform</span> <span class="p">{</span>
<span class="n">FunctionSpec</span> <span class="n">spec</span><span class="p">;</span>
<span class="k">repeated</span> <span class="kt">string</span> <span class="n">subtransforms</span><span class="p">;</span>
-
+
<span class="c1">// Maps from local string names to PCollection ids
</span> <span class="n">map</span><span class="o"><</span><span class="kt">string</span><span class="p">,</span> <span class="kt">bytes</span><span class="err">></span> <span class="n">inputs</span><span class="p">;</span>
<span class="n">map</span><span class="o"><</span><span class="kt">string</span><span class="p">,</span> <span class="kt">bytes</span><span class="err">></span> <span class="n">outputs</span><span class="p">;</span>
@@ -1289,7 +1333,7 @@ to JSON (or a proto <code class="highlighter-rouge">Struct</code>) and passed al
<span class="c1">// TODO: protocol for rejecting pipelines that cannot be executed
</span> <span class="c1">// by this runner. May just be REJECTED job state with error message.
-</span>
+</span>
<span class="c1">// totally opaque to the SDK; for the shim to interpret
</span> <span class="n">Any</span> <span class="n">contents</span><span class="p">;</span>
<span class="p">}</span>
@@ -1308,24 +1352,24 @@ depend upon wholly) querying metrics over this channel.</p>
<span class="kt">bytes</span> <span class="n">pipeline_id</span><span class="p">;</span>
<span class="o">...</span>
<span class="p">}</span>
-
+
<span class="kd">message</span> <span class="nc">GetStateRequest</span> <span class="p">{</span>
<span class="kt">bytes</span> <span class="n">pipeline_id</span><span class="p">;</span>
<span class="o">...</span>
<span class="p">}</span>
-
+
<span class="kd">message</span> <span class="nc">GetStateResponse</span> <span class="p">{</span>
<span class="n">JobState</span> <span class="n">state</span><span class="p">;</span>
<span class="o">...</span>
<span class="p">}</span>
-
+
<span class="kd">enum</span> <span class="n">JobState</span> <span class="p">{</span>
<span class="o">...</span>
<span class="p">}</span>
</code></pre>
</div>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/source-repository/index.html b/content/contribute/source-repository/index.html
index d997b77..ca2fa2d 100644
--- a/content/contribute/source-repository/index.html
+++ b/content/contribute/source-repository/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/source-repository/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,67 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-source-repository">Apache Beam Source Repository</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#web-browser-access">Web Browser Access</a></li>
+ <li><a href="#anonymous-access">Anonymous Access</a></li>
+ <li><a href="#developer-access">Developer Access</a></li>
+ <li><a href="#access-from-behind-a-firewall">Access from Behind a Firewall</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-source-repository">Apache Beam Source Repository</h1>
<p>This project uses <a href="http://git-scm.com">Git</a> to manage its source code.</p>
@@ -170,6 +163,7 @@
<h2 id="access-from-behind-a-firewall">Access from Behind a Firewall</h2>
<p>Refer to the documentation of the SCM used for more information about access behind a firewall.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/team/index.html b/content/contribute/team/index.html
index b9987a8..ba96eb6 100644
--- a/content/contribute/team/index.html
+++ b/content/contribute/team/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/team/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,64 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-team">Apache Beam Team</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#">2>Beam Committers</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-team">Apache Beam Team</h1>
<p>A successful project requires many people to play many roles. Some members write code or documentation, while others are valuable as testers, submitting patches and suggestions.</p>
@@ -430,6 +420,7 @@
</table>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/testing/index.html b/content/contribute/testing/index.html
index b6be34c..4ca57eb 100644
--- a/content/contribute/testing/index.html
+++ b/content/contribute/testing/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/testing/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,91 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="beam-testing-documentation">Beam Testing Documentation</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#overview">Overview</a></li>
+ <li><a href="#testing-matrix">Testing Matrix</a>
+ <ul>
+ <li><a href="#java-sdk">Java SDK</a></li>
+ <li><a href="#python-sdk">Python SDK</a></li>
+ </ul>
+ </li>
+ <li><a href="#testing-scenarios">Testing Scenarios</a>
+ <ul>
+ <li><a href="#precommit">Precommit</a></li>
+ <li><a href="#postcommit">Postcommit</a></li>
+ </ul>
+ </li>
+ <li><a href="#testing-types">Testing Types</a>
+ <ul>
+ <li><a href="#unit">Unit</a></li>
+ <li><a href="#validatesrunner">ValidatesRunner</a></li>
+ <li><a href="#e2e">E2E</a></li>
+ </ul>
+ </li>
+ <li><a href="#testing-systems">Testing Systems</a>
+ <ul>
+ <li><a href="#e2e-testing-framework">E2E Testing Framework</a></li>
+ <li><a href="#validatesrunner-tests">ValidatesRunner Tests</a></li>
+ <li><a href="#effective-use-of-the-testpipeline-junit-rule">Effective use of the TestPipeline JUnit rule</a></li>
+ <li><a href="#api-surface-testing">API Surface testing</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="beam-testing-documentation">Beam Testing Documentation</h1>
<ul id="markdown-toc">
<li><a href="#overview" id="markdown-toc-overview">Overview</a></li>
@@ -472,9 +489,9 @@ breakages. Beam Java unit tests are written in JUnit.</p>
<h4 id="how-to-run-needsrunner-tests">How to run NeedsRunner tests</h4>
<p>NeedsRunner is a category of tests that require a Beam runner. A subset of these
-tests cannot be executed while building their corresponding modules because all
-runners depend on these modules (e.g. <code class="highlighter-rouge">sdks/java/core</code>) to be built. To break
-the circular dependency, these tests are executed after the Direct Runner is
+tests cannot be executed while building their corresponding modules because all
+runners depend on these modules (e.g. <code class="highlighter-rouge">sdks/java/core</code>) to be built. To break
+the circular dependency, these tests are executed after the Direct Runner is
built.</p>
<p>To run this subset of the NeedsRunner tests (requires Maven 3.3.1+):</p>
@@ -492,7 +509,7 @@ $ mvn -pl runners/direct-java surefire:test@validates-runner-tests
<p>will run the <code class="highlighter-rouge">MapElementsTest.testMapBasic()</code> test.</p>
-<p>NeedsRunner tests in modules that are not required to build runners (e.g.
+<p>NeedsRunner tests in modules that are not required to build runners (e.g.
<code class="highlighter-rouge">sdks/java/io/jdbc</code>) can be executed with the <code class="highlighter-rouge">mvn test</code> command:</p>
<div class="highlighter-rouge"><pre class="highlight"><code>mvn -pl sdks/java/io/jdbc test -Dgroups=org.apache.beam.sdk.testing.NeedsRunner
@@ -565,28 +582,28 @@ verify that the simple pipelines they run end in the correct state.</p>
<h3 id="effective-use-of-the-testpipeline-junit-rule">Effective use of the TestPipeline JUnit rule</h3>
-<p><code class="highlighter-rouge">TestPipeline</code> is JUnit rule designed to facilitate testing pipelines.
-In combination with <code class="highlighter-rouge">PAssert</code>, the two can be used for testing and
-writing assertions over pipelines. However, in order for these assertions
-to be effective, the constructed pipeline <strong>must</strong> be run by a pipeline
-runner. If the pipeline is not run (i.e., executed) then the
-constructed <code class="highlighter-rouge">PAssert</code> statements will not be triggered, and will thus
+<p><code class="highlighter-rouge">TestPipeline</code> is JUnit rule designed to facilitate testing pipelines.
+In combination with <code class="highlighter-rouge">PAssert</code>, the two can be used for testing and
+writing assertions over pipelines. However, in order for these assertions
+to be effective, the constructed pipeline <strong>must</strong> be run by a pipeline
+runner. If the pipeline is not run (i.e., executed) then the
+constructed <code class="highlighter-rouge">PAssert</code> statements will not be triggered, and will thus
be ineffective.</p>
<p>To prevent such cases, <code class="highlighter-rouge">TestPipeline</code> has some protection mechanisms in place.</p>
<p><strong>Abandoned node detection (performed automatically)</strong></p>
-<p>Abandoned nodes are <code class="highlighter-rouge">PTransforms</code>, <code class="highlighter-rouge">PAsserts</code> included, that were not
-executed by the pipeline runner. Abandoned nodes are most likely to occur
+<p>Abandoned nodes are <code class="highlighter-rouge">PTransforms</code>, <code class="highlighter-rouge">PAsserts</code> included, that were not
+executed by the pipeline runner. Abandoned nodes are most likely to occur
due to the one of the following scenarios:</p>
<ol>
<li>Lack of a <code class="highlighter-rouge">pipeline.run()</code> statement at the end of a test.</li>
<li>Addition of <code class="highlighter-rouge">PTransform</code>s after the pipeline has already run.</li>
</ol>
-<p>Abandoned node detection is <em>automatically enabled</em> when a real pipeline
-runner (i.e. not a <code class="highlighter-rouge">CrashingRunner</code>) and/or a
+<p>Abandoned node detection is <em>automatically enabled</em> when a real pipeline
+runner (i.e. not a <code class="highlighter-rouge">CrashingRunner</code>) and/or a
<code class="highlighter-rouge">@NeedsRunner</code> / <code class="highlighter-rouge">@ValidatesRunner</code> annotation are detected.</p>
<p>Consider the following test:</p>
@@ -599,7 +616,7 @@ runner (i.e. not a <code class="highlighter-rouge">CrashingRunner</code>) and/or
<span class="nd">@Category</span><span class="o">(</span><span class="n">NeedsRunner</span><span class="o">.</span><span class="na">class</span><span class="o">)</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">myPipelineTest</span><span class="o">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span>
-<span class="kd">final</span> <span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">pCollection</span> <span class="o">=</span>
+<span class="kd">final</span> <span class="n">PCollection</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">pCollection</span> <span class="o">=</span>
<span class="n">pipeline</span>
<span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="s">"Create"</span><span class="o">,</span> <span class="n">Create</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">WORDS</span><span class="o">).</span><span class="na">withCoder</span><span class="o">(</span><span class="n">StringUtf8Coder</span><span class="o">.</span><span class="na">of</span><span class="o">()))</span>
<span class="o">.</span><span class="na">apply</span><span class="o">(</span>
@@ -612,7 +629,7 @@ runner (i.e. not a <code class="highlighter-rouge">CrashingRunner</code>) and/or
<span class="k">return</span> <span class="n">WHATEVER</span><span class="o">;</span>
<span class="o">}</span>
<span class="o">}));</span>
-
+
<span class="n">PAssert</span><span class="o">.</span><span class="na">that</span><span class="o">(</span><span class="n">pCollection</span><span class="o">).</span><span class="na">containsInAnyOrder</span><span class="o">(</span><span class="n">WHATEVER</span><span class="o">);</span>
<span class="cm">/* ERROR: pipeline.run() is missing, PAsserts are ineffective */</span>
@@ -624,12 +641,12 @@ runner (i.e. not a <code class="highlighter-rouge">CrashingRunner</code>) and/or
</code></pre>
</div>
-<p>The <code class="highlighter-rouge">PAssert</code> at the end of this test method will not be executed, since
-<code class="highlighter-rouge">pipeline</code> is never run, making this test ineffective. If this test method
-is run using an actual pipeline runner, an exception will be thrown
+<p>The <code class="highlighter-rouge">PAssert</code> at the end of this test method will not be executed, since
+<code class="highlighter-rouge">pipeline</code> is never run, making this test ineffective. If this test method
+is run using an actual pipeline runner, an exception will be thrown
indicating that there was no <code class="highlighter-rouge">run()</code> invocation in the test.</p>
-<p>Exceptions that are thrown prior to executing a pipeline, will fail
+<p>Exceptions that are thrown prior to executing a pipeline, will fail
the test unless handled by an <code class="highlighter-rouge">ExpectedException</code> rule.</p>
<p>Consider the following test:</p>
@@ -661,40 +678,40 @@ the test unless handled by an <code class="highlighter-rouge">ExpectedException<
</code></pre>
</div>
-<p>The application of the <code class="highlighter-rouge">read</code> transform throws an exception, which is then
-handled by the <code class="highlighter-rouge">thrown</code> <code class="highlighter-rouge">ExpectedException</code> rule.
-In light of this exception, the fact this test has abandoned nodes
-(the <code class="highlighter-rouge">read</code> transform) does not play a role since the test fails before
+<p>The application of the <code class="highlighter-rouge">read</code> transform throws an exception, which is then
+handled by the <code class="highlighter-rouge">thrown</code> <code class="highlighter-rouge">ExpectedException</code> rule.
+In light of this exception, the fact this test has abandoned nodes
+(the <code class="highlighter-rouge">read</code> transform) does not play a role since the test fails before
the pipeline would have been executed (had there been a <code class="highlighter-rouge">run()</code> statement).</p>
<p><strong>Auto-add <code class="highlighter-rouge">pipeline.run()</code> (disabled by default)</strong></p>
-<p>A <code class="highlighter-rouge">TestPipeline</code> instance can be configured to auto-add a missing <code class="highlighter-rouge">run()</code>
-statement by setting <code class="highlighter-rouge">testPipeline.enableAutoRunIfMissing(true/false)</code>.
-If this feature is enabled, no exception will be thrown in case of a
+<p>A <code class="highlighter-rouge">TestPipeline</code> instance can be configured to auto-add a missing <code class="highlighter-rouge">run()</code>
+statement by setting <code class="highlighter-rouge">testPipeline.enableAutoRunIfMissing(true/false)</code>.
+If this feature is enabled, no exception will be thrown in case of a
missing <code class="highlighter-rouge">run()</code> statement, instead, one will be added automatically.</p>
<h3 id="api-surface-testing">API Surface testing</h3>
-<p>The surface of an API is the set of public classes that are exposed to the
-outer world. In order to keep the API tight and avoid unnecessarily exposing
-classes, Beam provides the <code class="highlighter-rouge">ApiSurface</code> utility class.
-Using the <code class="highlighter-rouge">ApiSurface</code> class, we can assert the API surface against an
+<p>The surface of an API is the set of public classes that are exposed to the
+outer world. In order to keep the API tight and avoid unnecessarily exposing
+classes, Beam provides the <code class="highlighter-rouge">ApiSurface</code> utility class.
+Using the <code class="highlighter-rouge">ApiSurface</code> class, we can assert the API surface against an
expected set of classes.</p>
<p>Consider the following snippet:</p>
<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="nd">@Test</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">testMyApiSurface</span><span class="o">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span>
-
+
<span class="kd">final</span> <span class="n">Package</span> <span class="n">thisPackage</span> <span class="o">=</span> <span class="n">getClass</span><span class="o">().</span><span class="na">getPackage</span><span class="o">();</span>
<span class="kd">final</span> <span class="n">ClassLoader</span> <span class="n">thisClassLoader</span> <span class="o">=</span> <span class="n">getClass</span><span class="o">().</span><span class="na">getClassLoader</span><span class="o">();</span>
-
+
<span class="kd">final</span> <span class="n">ApiSurface</span> <span class="n">apiSurface</span> <span class="o">=</span>
<span class="n">ApiSurface</span><span class="o">.</span><span class="na">ofPackage</span><span class="o">(</span><span class="n">thisPackage</span><span class="o">,</span> <span class="n">thisClassLoader</span><span class="o">)</span>
<span class="o">.</span><span class="na">pruningPattern</span><span class="o">(</span><span class="s">"org[.]apache[.]beam[.].*Test.*"</span><span class="o">)</span>
<span class="o">.</span><span class="na">pruningPattern</span><span class="o">(</span><span class="s">"org[.]apache[.]beam[.].*IT"</span><span class="o">)</span>
<span class="o">.</span><span class="na">pruningPattern</span><span class="o">(</span><span class="s">"java[.]lang.*"</span><span class="o">);</span>
-
+
<span class="nd">@SuppressWarnings</span><span class="o">(</span><span class="s">"unchecked"</span><span class="o">)</span>
<span class="kd">final</span> <span class="n">Set</span><span class="o"><</span><span class="n">Matcher</span><span class="o"><</span><span class="n">Class</span><span class="o"><?>>></span> <span class="n">allowed</span> <span class="o">=</span>
<span class="n">ImmutableSet</span><span class="o">.</span><span class="na">of</span><span class="o">(</span>
@@ -702,7 +719,7 @@ expected set of classes.</p>
<span class="n">classesInPackage</span><span class="o">(</span><span class="s">"org.apache.beam.y"</span><span class="o">),</span>
<span class="n">classesInPackage</span><span class="o">(</span><span class="s">"org.apache.beam.z"</span><span class="o">),</span>
<span class="n">Matchers</span><span class="o">.<</span><span class="n">Class</span><span class="o"><?>></span><span class="n">equalTo</span><span class="o">(</span><span class="n">Other</span><span class="o">.</span><span class="na">class</span><span class="o">));</span>
-
+
<span class="n">assertThat</span><span class="o">(</span><span class="n">apiSurface</span><span class="o">,</span> <span class="n">containsOnlyClassesMatching</span><span class="o">(</span><span class="n">allowed</span><span class="o">));</span>
<span class="o">}</span>
</code></pre>
@@ -712,12 +729,13 @@ expected set of classes.</p>
</code></pre>
</div>
-<p>This test will fail if the classes exposed by <code class="highlighter-rouge">getClass().getPackage()</code>, except
+<p>This test will fail if the classes exposed by <code class="highlighter-rouge">getClass().getPackage()</code>, except
classes which reside under <code class="highlighter-rouge">"org[.]apache[.]beam[.].*Test.*"</code>,<br />
<code class="highlighter-rouge">"org[.]apache[.]beam[.].*IT"</code> or <code class="highlighter-rouge">"java[.]lang.*"</code>, belong to neither
-of the packages: <code class="highlighter-rouge">org.apache.beam.x</code>, <code class="highlighter-rouge">org.apache.beam.y</code>, <code class="highlighter-rouge">org.apache.beam.z</code>,
+of the packages: <code class="highlighter-rouge">org.apache.beam.x</code>, <code class="highlighter-rouge">org.apache.beam.y</code>, <code class="highlighter-rouge">org.apache.beam.z</code>,
nor equal to <code class="highlighter-rouge">Other.class</code>.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/contribute/work-in-progress/index.html b/content/contribute/work-in-progress/index.html
index 1a33c2e..0cbb43f 100644
--- a/content/contribute/work-in-progress/index.html
+++ b/content/contribute/work-in-progress/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/contribute/work-in-progress/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,65 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="work-in-progress-in-the-apache-beam-project">Work In Progress in the Apache Beam Project</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Contribute</span></li>
+<li><a href="/contribute">Get started contributing</a></li>
+<li>
+ <span class="section-nav-list-title">Guides</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing guide</a></li>
+ <li><a href="/contribute/release-guide/">Release guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform style guide</a></li>
+ <li><a href="/contribute/runner-guide/">Runner authoring guide</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Technical References</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/design-principles/">Design principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing projects</a></li>
+ <li><a href="/contribute/source-repository/">Source repository</a></li>
+ <li><a href="/contribute/docker-images/">Docker images</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Promotion</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/contribute/presentation-materials/">Presentation materials</a></li>
+ <li><a href="/contribute/logos/">Logos and design</a></li>
+ </ul>
+</li>
+<li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+<li><a href="/contribute/team/">Team</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#starter-tasks">Starter Tasks</a></li>
+ <li><a href="#feature-branches">Feature Branches</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="work-in-progress-in-the-apache-beam-project">Work In Progress in the Apache Beam Project</h1>
<p>As mentioned in the <a href="/contribute/contribution-guide/">Contribution Guide</a>, all work in Beam is tracked in our <a href="https://issues.apache.org/jira/browse/BEAM">Apache JIRA</a>. In addition, the following types of work may be of particular interest to the Beam community.</p>
@@ -206,7 +197,7 @@
</tbody>
</table>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/css/site.css b/content/css/site.css
index cd6d041..c384f82 100644
--- a/content/css/site.css
+++ b/content/css/site.css
@@ -6057,6 +6057,19 @@ div.cap-toggle {
.highlight .il {
color: #099; }
+.highlighter-custom {
+ border: 1px solid #ccc;
+ border-radius: 4px; }
+ .highlighter-custom pre {
+ background: #eef;
+ border-radius: 0;
+ border: 0;
+ margin: 0; }
+ .highlighter-custom p,
+ .highlighter-custom h2,
+ .highlighter-custom h3 {
+ margin: 10px; }
+
nav.language-switcher, nav.runner-switcher {
margin: 25px 0; }
nav.language-switcher ul, nav.runner-switcher ul {
@@ -6086,8 +6099,9 @@ body {
padding-top: 130px; }
.body:not(.body--index) .body__contained {
padding: 0 30px;
- max-width: 1280px;
- margin: 0 auto; }
+ max-width: 1280px; }
+ .body:not(.body--index) .body__contained figure img {
+ width: 100%; }
.section:not(.section--wide) {
padding-right: 30px;
@@ -6102,6 +6116,23 @@ body {
.section a {
color: #FF6D00; }
+.body__section-nav {
+ margin-left: 256px;
+ width: calc(100% - 492px); }
+ .body__section-nav > [id]:before {
+ content: "";
+ display: block;
+ height: 82px;
+ margin: -82px 0 0; }
+ @media (max-width: 1024px) {
+ .body__section-nav {
+ margin-left: 0;
+ width: 100%; } }
+
+.container-main-content {
+ padding: 0 20px;
+ position: relative; }
+
.navbar {
padding: 15px 0; }
.navbar .navbar-nav > li > a {
@@ -6121,8 +6152,50 @@ body {
color: #FF6D00; }
.navbar a span {
color: #555; }
- .navbar .navbar-toggle .icon-bar {
- background-color: #555; }
+ .navbar .navbar-toggle {
+ float: left; }
+ .navbar .navbar-toggle .icon-bar {
+ background-color: #555; }
+ @media (max-width: 1024px) {
+ .navbar .navbar-toggle {
+ display: block; } }
+ @media (max-width: 1024px) {
+ .navbar .navbar-container {
+ background-color: #FFF;
+ bottom: 0;
+ min-height: 100vh;
+ max-width: 256px;
+ padding: 15px;
+ position: fixed;
+ top: 0;
+ transition: transform 100ms linear;
+ width: calc(100% - 32px); }
+ .navbar .navbar-container .navbar-nav > li {
+ width: 100%; }
+ .navbar .navbar-container.closed {
+ transform: translateX(-100%); }
+ .navbar .navbar-container.open {
+ transform: translateX(0); } }
+ .navbar .navbar-mask {
+ background: #212121;
+ bottom: 0;
+ display: none;
+ min-height: 100vh;
+ position: fixed;
+ top: 0;
+ transition: opacity 200ms; }
+ @media (max-width: 1024px) {
+ .navbar .navbar-mask {
+ display: block; }
+ .navbar .navbar-mask.closed {
+ opacity: 0;
+ width: 0; }
+ .navbar .navbar-mask.open {
+ opacity: 0.5;
+ width: 100%; } }
+ @media (max-width: 1024px) {
+ .navbar .navbar-right {
+ margin-right: -15px; } }
.button {
-webkit-transition: background-color 0.3s;
@@ -6483,3 +6556,94 @@ body {
.pillars .pillars__cols .pillars__cols__col .pillars__cols__col__body {
padding: 0 30px;
margin: 0 auto; } }
+
+.section-nav {
+ max-width: 250px;
+ position: fixed;
+ width: 100%; }
+ .section-nav nav {
+ background-color: #F7F7F7;
+ padding: 15px;
+ max-height: calc(100vh - 130px);
+ overflow-y: auto; }
+ .section-nav nav ul {
+ list-style: none; }
+ .section-nav nav li {
+ margin-bottom: 5px; }
+ .section-nav nav li:last-child {
+ margin-bottom: 0; }
+ .section-nav nav a {
+ font-size: 12px; }
+ .section-nav nav > .section-nav-list {
+ padding: 0; }
+ .section-nav nav > .section-nav-list .section-nav-list {
+ padding-left: 15px; }
+ @media (max-width: 1024px) {
+ .section-nav nav {
+ background-color: #FFF;
+ max-height: 100vh; } }
+ .section-nav-back {
+ display: none;
+ padding: 15px; }
+ @media (max-width: 1024px) {
+ .section-nav-back {
+ display: block; } }
+ .section-nav-list-title {
+ display: block;
+ font-size: 12px;
+ margin-bottom: 5px;
+ padding-top: 5px; }
+ .section-nav-list-main-title {
+ display: block;
+ font-size: 12px;
+ font-weight: bold;
+ margin-bottom: 15px;
+ text-transform: uppercase; }
+ @media (max-width: 1024px) {
+ .section-nav {
+ background-color: #F7F7F7;
+ bottom: 0;
+ left: 0;
+ max-width: 256px;
+ position: fixed;
+ top: 0;
+ transition: transform 100ms linear;
+ width: calc(100% - 32px);
+ z-index: 10000; }
+ .section-nav nav {
+ height: calc(100vh - 44px);
+ overflow-y: auto; }
+ .section-nav.closed {
+ transform: translateX(-100%); }
+ .section-nav.open {
+ transform: translateX(0); } }
+
+.page-nav {
+ border-left: 3px solid #FF6D00;
+ overflow-y: auto;
+ max-height: calc(100vh - 130px);
+ padding: 0 15px;
+ position: fixed;
+ width: 240px; }
+ .page-nav li {
+ display: block;
+ width: 100%; }
+ .page-nav li a {
+ color: #37424B;
+ display: block;
+ font-size: 12px;
+ padding: 5px; }
+ .page-nav li span {
+ font-size: 12; }
+ .page-nav li ul {
+ padding-left: 20px; }
+ .page-nav .nav > li.active a {
+ text-decoration: underline; }
+ @media (max-width: 1024px) {
+ .page-nav {
+ margin-top: 0;
+ max-height: initial;
+ right: 0;
+ padding: 0 30px;
+ position: relative;
+ width: 100%; } }
diff --git a/content/documentation/dsls/sql/index.html b/content/documentation/dsls/sql/index.html
index f60509d..ab9188d 100644
--- a/content/documentation/dsls/sql/index.html
+++ b/content/documentation/dsls/sql/index.html
@@ -4,7 +4,7 @@
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>DSLs: SQL</title>
+ <title>Beam DSLs: SQL</title>
<meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow [...]
">
<link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400" rel="stylesheet">
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/dsls/sql/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,30 +77,88 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <ul>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">SDKS</span></li>
+<li><a href="/documentation/sdks/java/">Java SDK</a></li>
+<li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li><a href="/documentation/sdks/python/">Python SDK</a></li>
+<li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li>
+ <span class="section-nav-list-title">DSLs</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/dsls/sql/">SQL</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
<li><a href="#overview">1. Overview</a></li>
- <li><a href="#usage">2. Usage of DSL APIs</a></li>
+ <li><a href="#usage">2. Usage of DSL APIs</a>
+ <ul>
+ <li><a href="#beamrecord">BeamRecord</a></li>
+ <li><a href="#beamsql">BeamSql</a></li>
+ </ul>
+ </li>
<li><a href="#functionality">3. Functionality in Beam SQL</a>
<ul>
<li><a href="#features">3.1. Supported Features</a></li>
- <li><a href="#data-type">3.2. Data Types</a></li>
- <li><a href="#built-in-functions">3.3. built-in SQL functions</a></li>
+ <li><a href="#data-types">3.2. Data Types</a></li>
+ <li><a href="#built-in-functions">3.3. Built-in SQL functions</a></li>
+ </ul>
+ </li>
+ <li><a href="#internals-of-sql">4. Internals of Beam SQL</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="beam-sql">Beam SQL</h1>
+
+<ul id="markdown-toc">
+ <li><a href="#overview" id="markdown-toc-overview">1. Overview</a></li>
+ <li><a href="#usage" id="markdown-toc-usage">2. Usage of DSL APIs</a> <ul>
+ <li><a href="#beamrecord" id="markdown-toc-beamrecord">BeamRecord</a></li>
+ <li><a href="#beamsql" id="markdown-toc-beamsql">BeamSql</a></li>
+ </ul>
+ </li>
+ <li><a href="#functionality" id="markdown-toc-functionality">3. Functionality in Beam SQL</a> <ul>
+ <li><a href="#features" id="markdown-toc-features">3.1. Supported Features</a></li>
+ <li><a href="#data-types" id="markdown-toc-data-types">3.2. Data Types</a></li>
+ <li><a href="#built-in-functions" id="markdown-toc-built-in-functions">3.3. Built-in SQL functions</a></li>
</ul>
</li>
- <li><a href="#internal-of-sql">4. The Internal of Beam SQL</a></li>
+ <li><a href="#internals-of-sql" id="markdown-toc-internals-of-sql">4. Internals of Beam SQL</a></li>
</ul>
<p>This page describes the implementation of Beam SQL, and how to simplify a Beam pipeline with DSL APIs.</p>
<blockquote>
- <p>Note, Beam SQL hasn’t been merged to master branch yet(being developed with branch <a href="https://github.com/apache/beam/tree/DSL_SQL">DSL_SQL</a>), but is coming soon.</p>
+ <p>Note: Beam SQL hasn’t been merged to master branch yet (being developed with branch <a href="https://github.com/apache/beam/tree/DSL_SQL">DSL_SQL</a>), but is coming soon.</p>
</blockquote>
-<h1 id="a-nameoverviewa1-overview"><a name="overview"></a>1. Overview</h1>
+<h2 id="overview">1. Overview</h2>
+
<p>SQL is a well-adopted standard to process data with concise syntax. With DSL APIs (currently available only in Java), now <code class="highlighter-rouge">PCollection</code>s can be queried with standard SQL statements, like a regular table. The DSL APIs leverage <a href="http://calcite.apache.org/">Apache Calcite</a> to parse and optimize SQL queries, then translate into a composite Beam <code class="highlighter-rouge">PTransform</code>. In this way, both SQL and normal Beam <code cla [...]
<p>There are two main pieces to the SQL DSL API:</p>
@@ -175,11 +170,11 @@
<p>We’ll look at each of these below.</p>
-<h1 id="a-nameusagea2-usage-of-dsl-apis"><a name="usage"></a>2. Usage of DSL APIs</h1>
+<h2 id="usage">2. Usage of DSL APIs</h2>
-<h2 id="beamrecord">BeamRecord</h2>
+<h3 id="beamrecord">BeamRecord</h3>
-<p>Before applying a SQL query to a <code class="highlighter-rouge">PCollection</code>, the data in the collection must be in <code class="highlighter-rouge">BeamRecord</code> format. A <code class="highlighter-rouge">BeamRecord</code> represents a single, immutable row in a Beam SQL <code class="highlighter-rouge">PCollection</code>. The names and types of the fields/columns in the record are defined by its associated <a href="/documentation/sdks/javadoc/2.1.0/index.html?org/apache/beam [...]
+<p>Before applying a SQL query to a <code class="highlighter-rouge">PCollection</code>, the data in the collection must be in <code class="highlighter-rouge">BeamRecord</code> format. A <code class="highlighter-rouge">BeamRecord</code> represents a single, immutable row in a Beam SQL <code class="highlighter-rouge">PCollection</code>. The names and types of the fields/columns in the record are defined by its associated <a href="/documentation/sdks/javadoc/2.1.0/index.html?org/apache/beam [...]
<p>A <code class="highlighter-rouge">PCollection<BeamRecord></code> can be created explicitly or implicitly:</p>
@@ -234,7 +229,7 @@ PCollection<BeamRecord> apps = pojos.apply(
<p>Once you have a <code class="highlighter-rouge">PCollection<BeamRecord></code> in hand, you may use the <code class="highlighter-rouge">BeamSql</code> APIs to apply SQL queries to it.</p>
-<h2 id="beamsql">BeamSql</h2>
+<h3 id="beamsql">BeamSql</h3>
<p><code class="highlighter-rouge">BeamSql</code> provides two methods for generating a <code class="highlighter-rouge">PTransform</code> from a SQL query, both of which are equivalent except for the number of inputs they support:</p>
@@ -268,7 +263,7 @@ PCollection<BeamRecord> output = namesAndFoods.apply(
<p><a href="https://github.com/apache/beam/blob/DSL_SQL/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlExample.java">BeamSqlExample</a> in the code repository shows basic usage of both APIs.</p>
-<h1 id="a-namefunctionalitya3-functionality-in-beam-sql"><a name="functionality"></a>3. Functionality in Beam SQL</h1>
+<h2 id="functionality">3. Functionality in Beam SQL</h2>
<p>Just as the unified model for both bounded and unbounded data in Beam, SQL DSL provides the same functionalities for bounded and unbounded <code class="highlighter-rouge">PCollection</code> as well. Here’s the supported SQL grammar supported in <a href="http://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form">BNF</a>-like form. An <code class="highlighter-rouge">UnsupportedOperationException</code> is thrown for unsupported features.</p>
<div class="highlighter-rouge"><pre class="highlight"><code>query:
@@ -302,7 +297,7 @@ tableExpression:
booleanExpression:
expression [ IS NULL | IS NOT NULL ]
| expression [ > | >= | = | < | <= | <> ] expression
- | booleanExpression [ AND | OR ] booleanExpression
+ | booleanExpression [ AND | OR ] booleanExpression
| NOT booleanExpression
| '(' booleanExpression ')'
@@ -325,7 +320,7 @@ groupItem:
</code></pre>
</div>
-<h2 id="a-namefeaturesa31-supported-features"><a name="features"></a>3.1. Supported Features</h2>
+<h3 id="features">3.1. Supported Features</h3>
<p><strong>1. aggregations;</strong></p>
@@ -453,7 +448,7 @@ PCollection<BeamSqlRow> result =
</code></pre>
</div>
-<h2 id="a-namedata-typea32-data-types"><a name="data-type"></a>3.2. Data Types</h2>
+<h3 id="data-types">3.2. Data Types</h3>
<p>Each type in Beam SQL maps to a Java class to holds the value in <code class="highlighter-rouge">BeamRecord</code>. The following table lists the relation between SQL types and Java classes, which are supported in current repository:</p>
<table class="table">
@@ -503,7 +498,7 @@ PCollection<BeamSqlRow> result =
</tbody>
</table>
-<h2 id="a-namebuilt-in-functionsa33-built-in-sql-functions"><a name="built-in-functions"></a>3.3. built-in SQL functions</h2>
+<h3 id="built-in-functions">3.3. Built-in SQL functions</h3>
<p>Beam SQL has implemented lots of build-in functions defined in <a href="http://calcite.apache.org">Apache Calcite</a>. The available functions are listed as below:</p>
@@ -901,7 +896,7 @@ PCollection<BeamSqlRow> result =
</tbody>
</table>
-<h1 id="a-nameinternal-of-sqla4-the-internal-of-beam-sql"><a name="internal-of-sql"></a>4. The Internal of Beam SQL</h1>
+<h2 id="internals-of-sql">4. Internals of Beam SQL</h2>
<p>Figure 1 describes the back-end steps from a SQL statement to a Beam <code class="highlighter-rouge">PTransform</code>.</p>
<p><img src="/images/beam_sql_dsl_workflow.png" alt="Workflow of Beam SQL DSL" title="workflow of Beam SQL DSL" /></p>
@@ -938,7 +933,7 @@ PCollection<BeamSqlRow> result =
</code></pre>
</div>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/execution-model/index.html b/content/documentation/execution-model/index.html
index 8afa172..26d7cce 100644
--- a/content/documentation/execution-model/index.html
+++ b/content/documentation/execution-model/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/execution-model/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,139 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-execution-model">Apache Beam Execution Model</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#processing-of-elements">Processing of elements</a>
+ <ul>
+ <li><a href="#serialization-and-communication">Serialization and communication</a></li>
+ <li><a href="#bundling-and-persistence">Bundling and persistence</a></li>
+ </ul>
+ </li>
+ <li><a href="#parallelism">Failures and parallelism within and between transforms</a>
+ <ul>
+ <li><a href="#data-parallelism">Data-parallelism within one transform</a></li>
+ <li><a href="#dependent-parallellism">Dependent-parallelism between transforms</a></li>
+ <li><a href="#failures-within-one-transform">Failures within one transform</a></li>
+ <li><a href="#coupled-failure">Coupled failure: Failures between transforms</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-execution-model">Apache Beam Execution Model</h1>
<p>The Beam model allows runners to execute your pipeline in different ways. You
may observe various effects as a result of the runner’s choices. This page
@@ -345,7 +410,7 @@ down since they aren’t following the normal <code class="highlighter-rouge">Do
<p>Executing transforms this way allows a runner to avoid persisting elements
between transforms, saving on persistence costs.</p>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/index.html b/content/documentation/index.html
index fc88d07..ee11f8b 100644
--- a/content/documentation/index.html
+++ b/content/documentation/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,134 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-documentation">Apache Beam Documentation</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#concepts">Concepts</a></li>
+ <li><a href="#pipeline-fundamentals">Pipeline Fundamentals</a></li>
+ <li><a href="#sdks">SDKs</a></li>
+ <li><a href="#runners">Runners</a>
+ <ul>
+ <li><a href="#available-runners">Available Runners</a></li>
+ <li><a href="#choosing-a-runner">Choosing a Runner</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-documentation">Apache Beam Documentation</h1>
<p>This section provides in-depth conceptual information and reference material for the Beam Model, SDKs, and Runners:</p>
@@ -196,6 +256,7 @@
<p>Once you have chosen which runner to use, see that runner’s page for more information about any initial runner-specific setup as well as any required or optional <code class="highlighter-rouge">PipelineOptions</code> for configuring it’s execution. You may also want to refer back to the Quickstart for <a href="/get-started/quickstart-java">Java</a> or <a href="/get-started/quickstart-py">Python</a> for instructions on executing the sample WordCount pipeline.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/io/authoring-java/index.html b/content/documentation/io/authoring-java/index.html
index 00a11df..cf2fb8c 100644
--- a/content/documentation/io/authoring-java/index.html
+++ b/content/documentation/io/authoring-java/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/io/authoring-java/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,126 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#example-io-transforms">Example I/O Transforms</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
<h1 id="authoring-io-transforms---java">Authoring I/O Transforms - Java</h1>
@@ -165,6 +217,7 @@ styles. These transforms are good examples to follow:</p>
<p><a href="/documentation/io/testing/">Testing I/O Transforms</a></p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/io/authoring-overview/index.html b/content/documentation/io/authoring-overview/index.html
index 4162b89..b913c69 100644
--- a/content/documentation/io/authoring-overview/index.html
+++ b/content/documentation/io/authoring-overview/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/io/authoring-overview/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,137 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#introduction">Introduction</a></li>
+ <li><a href="#suggested-steps-for-implementers">Suggested steps for implementers</a></li>
+ <li><a href="#read-transforms">Read transforms</a>
+ <ul>
+ <li><a href="#when-to-implement-using-the-source-api">When to implement using the <code class="highlighter-rouge">Source</code> API</a></li>
+ </ul>
+ </li>
+ <li><a href="#write-transforms">Write transforms</a>
+ <ul>
+ <li><a href="#when-to-implement-using-the-sink-api">When to implement using the <code class="highlighter-rouge">Sink</code> API</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
<h1 id="authoring-io-transforms---overview">Authoring I/O Transforms - Overview</h1>
@@ -251,6 +314,7 @@ For more details on actual implementation, continue with one of the the language
* [Authoring I/O Transforms - Java](/documentation/io/authoring-java/)
-->
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/io/authoring-python/index.html b/content/documentation/io/authoring-python/index.html
index 2e322f8..121d47c 100644
--- a/content/documentation/io/authoring-python/index.html
+++ b/content/documentation/io/authoring-python/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/io/authoring-python/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,123 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
<h1 id="authoring-io-transforms---python">Authoring I/O Transforms - Python</h1>
@@ -158,6 +207,7 @@
<p><a href="/documentation/io/testing/">Testing I/O Transforms</a></p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/io/built-in/hadoop/index.html b/content/documentation/io/built-in/hadoop/index.html
index 462054d..80c3ef2 100644
--- a/content/documentation/io/built-in/hadoop/index.html
+++ b/content/documentation/io/built-in/hadoop/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/io/built-in/hadoop/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,130 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#reading-using-hadoop-inputformat-io">Reading using Hadoop InputFormat IO</a></li>
+ <li><a href="#cassandra---cqlinputformat">Cassandra - CqlInputFormat</a></li>
+ <li><a href="#elasticsearch---esinputformat">Elasticsearch - EsInputFormat</a></li>
+ <li><a href="#hcatalog---hcatinputformat">HCatalog - HCatInputFormat</a></li>
+ <li><a href="#amazon-dynamodb---dynamodbinputformat">Amazon DynamoDB - DynamoDBInputFormat</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
<h1 id="hadoop-inputformat-io">Hadoop InputFormat IO</h1>
@@ -372,7 +428,7 @@
<h3 id="amazon-dynamodb---dynamodbinputformat">Amazon DynamoDB - DynamoDBInputFormat</h3>
<p>To read data from Amazon DynamoDB, use <code class="highlighter-rouge">org.apache.hadoop.dynamodb.read.DynamoDBInputFormat</code>.
-DynamoDBInputFormat implements the older <code class="highlighter-rouge">org.apache.hadoop.mapred.InputFormat</code> interface and to make it compatible with HadoopInputFormatIO which uses the newer abstract class <code class="highlighter-rouge">org.apache.hadoop.mapreduce.InputFormat</code>,
+DynamoDBInputFormat implements the older <code class="highlighter-rouge">org.apache.hadoop.mapred.InputFormat</code> interface and to make it compatible with HadoopInputFormatIO which uses the newer abstract class <code class="highlighter-rouge">org.apache.hadoop.mapreduce.InputFormat</code>,
a wrapper API is required which acts as an adapter between HadoopInputFormatIO and DynamoDBInputFormat (or in general any InputFormat implementing <code class="highlighter-rouge">org.apache.hadoop.mapred.InputFormat</code>)
The below example uses one such available wrapper API - <a href="https://github.com/twitter/elephant-bird/blob/master/core/src/main/java/com/twitter/elephantbird/mapreduce/input/MapReduceInputFormatWrapper.java">https://github.com/twitter/elephant-bird/blob/master/core/src/main/java/com/twitter/elephantbird/mapreduce/input/MapReduceInputFormatWrapper.java</a></p>
@@ -411,6 +467,7 @@ The below example uses one such available wrapper API - <a href="https://github.
</code></pre>
</div>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/io/built-in/index.html b/content/documentation/io/built-in/index.html
index 312e09c..4877274 100644
--- a/content/documentation/io/built-in/index.html
+++ b/content/documentation/io/built-in/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/io/built-in/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,123 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
<h1 id="built-in-io-transforms">Built-in I/O Transforms</h1>
@@ -270,6 +319,7 @@
</tr>
</table>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/io/contributing/index.html b/content/documentation/io/contributing/index.html
index 708763c..a845a5c 100644
--- a/content/documentation/io/contributing/index.html
+++ b/content/documentation/io/contributing/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/io/contributing/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,123 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
<h1 id="contributing-io-transforms">Contributing I/O Transforms</h1>
@@ -158,6 +207,7 @@
<p>Note: This guide is still in progress. There is an open issue to finish the guide: <a href="https://issues.apache.org/jira/browse/BEAM-1025">BEAM-1025</a>.</p>
</blockquote>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/io/io-toc/index.html b/content/documentation/io/io-toc/index.html
index 8f71794..721c097 100644
--- a/content/documentation/io/io-toc/index.html
+++ b/content/documentation/io/io-toc/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/io/io-toc/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,127 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="pipeline-io">Pipeline I/O</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#using-pipeline-io">Using Pipeline I/O</a></li>
+ <li><a href="#authoring-read-amp-write-io-transforms">Authoring Read & Write I/O Transforms</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="pipeline-io">Pipeline I/O</h1>
<h2 id="using-pipeline-io">Using Pipeline I/O</h2>
<ul>
@@ -170,6 +223,7 @@
* [Contributing I/O Transforms](/documentation/io/contributing/)
-->
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/io/testing/index.html b/content/documentation/io/testing/index.html
index 34af0a5..bb0b1b5 100644
--- a/content/documentation/io/testing/index.html
+++ b/content/documentation/io/testing/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/io/testing/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,148 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#testing-io-transforms-in-apache-beam">Testing I/O Transforms in Apache Beam</a></li>
+ <li><a href="#introduction">Introduction</a></li>
+ <li><a href="#a-note-on-performance-benchmarking">A note on performance benchmarking</a></li>
+ <li><a href="#test-balance-unit-vs-integration">Test Balance - Unit vs Integration</a></li>
+ <li><a href="#examples">Examples</a></li>
+ <li><a href="#unit-tests">Unit Tests</a>
+ <ul>
+ <li><a href="#goals">Goals</a></li>
+ <li><a href="#non-goals">Non-goals</a></li>
+ <li><a href="#implementing-unit-tests">Implementing unit tests</a></li>
+ <li><a href="#use-fakes">Use fakes</a></li>
+ <li><a href="#network-failure">Network failure</a></li>
+ </ul>
+ </li>
+ <li><a href="#batching">Batching</a></li>
+ <li><a href="#i-o-transform-integration-tests">I/O Transform Integration Tests</a>
+ <ul>
+ <li><a href="#it-goals">Goals</a></li>
+ <li><a href="#integration-tests-data-stores-and-kubernetes">Integration tests, data stores, and Kubernetes</a></li>
+ <li><a href="#running-integration-tests">Running integration tests</a></li>
+ <li><a href="#implementing-integration-tests">Implementing Integration Tests</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <p><a href="/documentation/io/io-toc/">Pipeline I/O Table of Contents</a></p>
<h2 id="testing-io-transforms-in-apache-beam">Testing I/O Transforms in Apache Beam</h2>
@@ -307,7 +381,7 @@
</div>
<p>Example run with the Cloud Dataflow runner:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>mvn verify -Dio-it-suite -pl sdks/java/io/jdbc -DintegrationTestPipelineOptions=["--project=PROJECT","--gcpTempLocation=GSBUCKET"] -DintegrationTestRunner=dataflow -DpkbLocation="/Users/me/dev/PerfKitBenchmarker/pkb.py"
+<div class="highlighter-rouge"><pre class="highlight"><code>mvn verify -Dio-it-suite -pl sdks/java/io/jdbc -DintegrationTestPipelineOptions=["--project=PROJECT","--gcpTempLocation=GSBUCKET"] -DintegrationTestRunner=dataflow -DpkbLocation="/Users/me/dev/PerfKitBenchmarker/pkb.py"
</code></pre>
</div>
@@ -768,6 +842,7 @@ If you have a well tested I/O transform, why not contribute it to Apache Beam? R
[Contributing I/O Transforms](/documentation/io/contributing/)
-->
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/pipelines/create-your-pipeline/index.html b/content/documentation/pipelines/create-your-pipeline/index.html
index 6c3a756..ac8160f 100644
--- a/content/documentation/pipelines/create-your-pipeline/index.html
+++ b/content/documentation/pipelines/create-your-pipeline/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/pipelines/create-your-pipeline/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,131 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="create-your-pipeline">Create Your Pipeline</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#creating-your-pipeline-object">Creating Your Pipeline Object</a></li>
+ <li><a href="#reading-data-into-your-pipeline">Reading Data Into Your Pipeline</a></li>
+ <li><a href="#applying-transforms-to-process-pipeline-data">Applying Transforms to Process Pipeline Data</a></li>
+ <li><a href="#writing-or-outputting-your-final-pipeline-data">Writing or Outputting Your Final Pipeline Data</a></li>
+ <li><a href="#running-your-pipeline">Running Your Pipeline</a></li>
+ <li><a href="#whats-next">What’s next</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="create-your-pipeline">Create Your Pipeline</h1>
<ul id="markdown-toc">
<li><a href="#creating-your-pipeline-object" id="markdown-toc-creating-your-pipeline-object">Creating Your Pipeline Object</a></li>
@@ -241,6 +298,7 @@
<li><a href="/documentation/pipelines/test-your-pipeline">Test your pipeline</a>.</li>
</ul>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/pipelines/design-your-pipeline/index.html b/content/documentation/pipelines/design-your-pipeline/index.html
index e44c441..7b74eec 100644
--- a/content/documentation/pipelines/design-your-pipeline/index.html
+++ b/content/documentation/pipelines/design-your-pipeline/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/pipelines/design-your-pipeline/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,136 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="design-your-pipeline">Design Your Pipeline</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#what-to-consider-when-designing-your-pipeline">What to consider when designing your pipeline</a></li>
+ <li><a href="#a-basic-pipeline">A basic pipeline</a></li>
+ <li><a href="#branching-pcollections">Branching PCollections</a>
+ <ul>
+ <li><a href="#multiple-transforms-process-the-same-pcollection">Multiple transforms process the same PCollection</a></li>
+ <li><a href="#a-single-transform-that-produces-multiple-outputs">A single transform that produces multiple outputs</a></li>
+ </ul>
+ </li>
+ <li><a href="#merging-pcollections">Merging PCollections</a></li>
+ <li><a href="#multiple-sources">Multiple sources</a></li>
+ <li><a href="#whats-next">What’s next</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="design-your-pipeline">Design Your Pipeline</h1>
<ul id="markdown-toc">
<li><a href="#what-to-consider-when-designing-your-pipeline" id="markdown-toc-what-to-consider-when-designing-your-pipeline">What to consider when designing your pipeline</a></li>
@@ -338,6 +400,7 @@
<li><a href="/documentation/pipelines/test-your-pipeline">Test your pipeline</a>.</li>
</ul>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/pipelines/test-your-pipeline/index.html b/content/documentation/pipelines/test-your-pipeline/index.html
index 0a72c9b..ae5d1a8 100644
--- a/content/documentation/pipelines/test-your-pipeline/index.html
+++ b/content/documentation/pipelines/test-your-pipeline/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/pipelines/test-your-pipeline/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,145 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="test-your-pipeline">Test Your Pipeline</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#testing-individual-dofn-objects">Testing Individual DoFn Objects</a>
+ <ul>
+ <li><a href="#creating-a-dofntester">Creating a DoFnTester</a></li>
+ <li><a href="#creating-test-inputs">Creating Test Inputs</a></li>
+ <li><a href="#processing-test-inputs-and-checking-results">Processing Test Inputs and Checking Results</a></li>
+ </ul>
+ </li>
+ <li><a href="#testing-composite-transforms">Testing Composite Transforms</a>
+ <ul>
+ <li><a href="#testpipeline">TestPipeline</a></li>
+ <li><a href="#using-the-create-transform">Using the Create Transform</a></li>
+ <li><a href="#passert">PAssert</a></li>
+ <li><a href="#an-example-test-for-a-composite-transform">An Example Test for a Composite Transform</a></li>
+ </ul>
+ </li>
+ <li><a href="#testing-a-pipeline-end-to-end">Testing a Pipeline End-to-End</a>
+ <ul>
+ <li><a href="#testing-the-wordcount-pipeline">Testing the WordCount Pipeline</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="test-your-pipeline">Test Your Pipeline</h1>
<ul id="markdown-toc">
<li><a href="#testing-individual-dofn-objects" id="markdown-toc-testing-individual-dofn-objects">Testing Individual DoFn Objects</a> <ul>
@@ -446,6 +517,7 @@ for the <code class="highlighter-rouge">DoFnTester</code> as follows:</p>
</code></pre>
</div>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/programming-guide/index.html b/content/documentation/programming-guide/index.html
index cb36205..b00ea56 100644
--- a/content/documentation/programming-guide/index.html
+++ b/content/documentation/programming-guide/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/programming-guide/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,179 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-programming-guide">Apache Beam Programming Guide</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#overview">1. Overview</a></li>
+ <li><a href="#creating-a-pipeline">2. Creating a pipeline</a>
+ <ul>
+ <li><a href="#configuring-pipeline-options">2.1. Configuring pipeline options</a></li>
+ </ul>
+ </li>
+ <li><a href="#pcollections">3. PCollections</a>
+ <ul>
+ <li><a href="#creating-a-pcollection">3.1. Creating a PCollection</a></li>
+ <li><a href="#pcollection-characteristics">3.2. PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li><a href="#transforms">4. Transforms</a>
+ <ul>
+ <li><a href="#applying-transforms">4.1. Applying transforms</a></li>
+ <li><a href="#core-beam-transforms">4.2. Core Beam transforms</a></li>
+ <li><a href="#requirements-for-writing-user-code-for-beam-transforms">4.3. Requirements for writing user code for Beam transforms</a></li>
+ <li><a href="#side-inputs">4.4. Side inputs</a></li>
+ <li><a href="#additional-outputs">4.5. Additional outputs</a></li>
+ <li><a href="#composite-transforms">4.6. Composite transforms</a></li>
+ </ul>
+ </li>
+ <li><a href="#pipeline-io">5. Pipeline I/O</a>
+ <ul>
+ <li><a href="#reading-input-data">5.1. Reading input data</a></li>
+ <li><a href="#writing-output-data">5.2. Writing output data</a></li>
+ <li><a href="#file-based-input-and-output-data">5.3. File-based input and output data</a></li>
+ <li><a href="#beam-provided-io-transforms">5.4. Beam-provided I/O transforms</a></li>
+ </ul>
+ </li>
+ <li><a href="#data-encoding-and-type-safety">6. Data encoding and type safety</a>
+ <ul>
+ <li><a href="#specifying-coders">6.1. Specifying coders</a></li>
+ <li><a href="#default-coders-and-the-coderregistry">6.2. Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li><a href="#windowing">7. Windowing</a>
+ <ul>
+ <li><a href="#windowing-basics">7.1. Windowing basics</a></li>
+ <li><a href="#provided-windowing-functions">7.2. Provided windowing functions</a></li>
+ <li><a href="#setting-your-pcollections-windowing-function">7.3. Setting your PCollection’s windowing function</a></li>
+ <li><a href="#watermarks-and-late-data">7.4. Watermarks and late data</a></li>
+ <li><a href="#adding-timestamps-to-a-pcollections-elements">7.5. Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li><a href="#triggers">8. Triggers</a>
+ <ul>
+ <li><a href="#event-time-triggers">8.1. Event time triggers</a></li>
+ <li><a href="#processing-time-triggers">8.2. Processing time triggers</a></li>
+ <li><a href="#data-driven-triggers">8.3. Data-driven triggers</a></li>
+ <li><a href="#setting-a-trigger">8.4. Setting a trigger</a></li>
+ <li><a href="#composite-triggers">8.5. Composite triggers</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-programming-guide">Apache Beam Programming Guide</h1>
<p>The <strong>Beam Programming Guide</strong> is intended for Beam users who want to use the
Beam SDKs to create data processing pipelines. It provides guidance for using
@@ -3100,6 +3205,7 @@ elements, or after a minute.</p>
</code></pre>
</div>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/resources/index.html b/content/documentation/resources/index.html
index 0477280..d72422c 100644
--- a/content/documentation/resources/index.html
+++ b/content/documentation/resources/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/resources/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,147 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="additional-resources-for-learning-about-apache-beam">Additional Resources for Learning about Apache Beam</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#general">General</a>
+ <ul>
+ <li><a href="#fundamentals-of-stream-processing-with-apache-beam">Fundamentals of Stream Processing with Apache Beam</a></li>
+ <li><a href="#apache-beam-a-unified-model-for-batch-and-streaming-data-processing">Apache Beam: A Unified Model for Batch and Streaming Data Processing</a></li>
+ <li><a href="#fundamentals-of-stream-processing-with-apache-beam-1">Fundamentals of Stream Processing with Apache Beam</a></li>
+ <li><a href="#the-world-beyond-batch-streaming-101">The World Beyond Batch: Streaming 101</a></li>
+ <li><a href="#the-world-beyond-batch-streaming-102">The World Beyond Batch: Streaming 102</a></li>
+ <li><a href="#software-engineering-radio-podcast-episode-272-apache-beam">Software Engineering Radio Podcast Episode 272: Apache Beam</a></li>
+ </ul>
+ </li>
+ <li><a href="#beam--friends">Beam & Friends</a>
+ <ul>
+ <li><a href="#flink-and-beam-current-state--roadmap">Flink and Beam: Current State & Roadmap</a></li>
+ <li><a href="#lessons-learned-from-developing-a-stream-processing-platform-at-scale">Lessons learned from developing a stream processing platform at scale</a></li>
+ </ul>
+ </li>
+ <li><a href="#technical-details">Technical Details</a>
+ <ul>
+ <li><a href="#watermarks-time-and-progress-in-apache-beam-and-beyond">Watermarks: Time and Progress in Apache Beam and Beyond</a></li>
+ <li><a href="#triggers-in-apache-beam">Triggers in Apache Beam</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="additional-resources-for-learning-about-apache-beam">Additional Resources for Learning about Apache Beam</h1>
<p>This page provides links to some of our favorite videos, podcasts, and articles that will help you get started and learn more about Apache Beam.</p>
@@ -269,6 +342,7 @@
<iframe width="560" height="315" src="https://www.youtube.com/embed/E1k0B9LN46M" frameborder="0" allowfullscreen=""></iframe>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/runners/apex/index.html b/content/documentation/runners/apex/index.html
index 34051b1..17cefd0 100644
--- a/content/documentation/runners/apex/index.html
+++ b/content/documentation/runners/apex/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/runners/apex/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,42 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="using-the-apache-apex-runner">Using the Apache Apex Runner</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Runners</span></li>
+<li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
+<li><a href="/documentation/runners/direct/">Direct Runner</a></li>
+<li><a href="/documentation/runners/apex/">Apache Apex</a></li>
+<li><a href="/documentation/runners/flink/">Apache Flink</a></li>
+<li><a href="/documentation/runners/gearpump/">Apache Gearpump</a></li>
+<li><a href="/documentation/runners/spark/">Apache Spark</a></li>
+<li><a href="/documentation/runners/dataflow/">Google Cloud Dataflow</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#apex-runner-prerequisites">Apex Runner prerequisites</a></li>
+ <li><a href="#running-wordcount-using-apex-runner">Running wordcount using Apex Runner</a></li>
+ <li><a href="#checking-output">Checking output</a></li>
+ <li><a href="#montoring-progress-of-your-job">Montoring progress of your job</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="using-the-apache-apex-runner">Using the Apache Apex Runner</h1>
<p>The Apex Runner executes Apache Beam pipelines using <a href="http://apex.apache.org/">Apache Apex</a> as an underlying engine. The runner has broad support for the <a href="/documentation/runners/capability-matrix/">Beam model and supports streaming and batch pipelines</a>.</p>
@@ -204,7 +172,7 @@ apex.application.*.operator.*.attr.TIMEOUT_WINDOW_COUNT=1200
<li>Apex command-line interface: <a href="http://apex.apache.org/docs/apex/apex_cli/#apex-cli-commands">Using the Apex CLI to get running application information</a>.</li>
</ul>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/runners/capability-matrix/index.html b/content/documentation/runners/capability-matrix/index.html
index 09f63a4..f834b25 100644
--- a/content/documentation/runners/capability-matrix/index.html
+++ b/content/documentation/runners/capability-matrix/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/runners/capability-matrix/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,36 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="beam-capability-matrix">Beam Capability Matrix</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Runners</span></li>
+<li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
+<li><a href="/documentation/runners/direct/">Direct Runner</a></li>
+<li><a href="/documentation/runners/apex/">Apache Apex</a></li>
+<li><a href="/documentation/runners/flink/">Apache Flink</a></li>
+<li><a href="/documentation/runners/gearpump/">Apache Gearpump</a></li>
+<li><a href="/documentation/runners/spark/">Apache Spark</a></li>
+<li><a href="/documentation/runners/dataflow/">Google Cloud Dataflow</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="beam-capability-matrix">Beam Capability Matrix</h1>
<p>Apache Beam provides a portable API layer for building sophisticated data-parallel processing pipelines that may be executed across a diversity of execution engines, or <i>runners</i>. The core concepts of this layer are based upon the Beam Model (formerly referred to as the <a href="http://www.vldb.org/pvldb/vol8/p1792-Akidau.pdf">Dataflow Model</a>), and implemented to varying degrees in each Beam runner. To help clarify the capabilities of individual runners, we’ve created the capa [...]
<p>Individual capabilities have been grouped by their corresponding <span class="wwwh-what-dark">What</span> / <span class="wwwh-where-dark">Where</span> / <span class="wwwh-when-dark">When</span> / <span class="wwwh-how-dark">How</span> question:</p>
@@ -2943,6 +2905,7 @@
</center>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/runners/dataflow/index.html b/content/documentation/runners/dataflow/index.html
index 44e474c..8ab9199 100644
--- a/content/documentation/runners/dataflow/index.html
+++ b/content/documentation/runners/dataflow/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/runners/dataflow/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,52 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="using-the-google-cloud-dataflow-runner">Using the Google Cloud Dataflow Runner</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Runners</span></li>
+<li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
+<li><a href="/documentation/runners/direct/">Direct Runner</a></li>
+<li><a href="/documentation/runners/apex/">Apache Apex</a></li>
+<li><a href="/documentation/runners/flink/">Apache Flink</a></li>
+<li><a href="/documentation/runners/gearpump/">Apache Gearpump</a></li>
+<li><a href="/documentation/runners/spark/">Apache Spark</a></li>
+<li><a href="/documentation/runners/dataflow/">Google Cloud Dataflow</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#cloud-dataflow-runner-prerequisites-and-setup">Cloud Dataflow Runner prerequisites and setup</a>
+ <ul>
+ <li><a href="#specify-your-dependency">Specify your dependency</a></li>
+ <li><a href="#authentication">Authentication</a></li>
+ </ul>
+ </li>
+ <li><a href="#pipeline-options-for-the-cloud-dataflow-runner">Pipeline options for the Cloud Dataflow Runner</a></li>
+ <li><a href="#additional-information-and-caveats">Additional information and caveats</a>
+ <ul>
+ <li><a href="#monitoring-your-job">Monitoring your job</a></li>
+ <li><a href="#blocking-execution">Blocking Execution</a></li>
+ <li><a href="#streaming-execution">Streaming Execution</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="using-the-google-cloud-dataflow-runner">Using the Google Cloud Dataflow Runner</h1>
<nav class="language-switcher">
<strong>Adapt for:</strong>
@@ -327,7 +305,7 @@ interface (and any subinterfaces) for additional pipeline configuration options.
<p><span class="language-java">If your pipeline uses an unbounded data source or sink, you must set the <code class="highlighter-rouge">streaming</code> option to <code class="highlighter-rouge">true</code>.</span>
<span class="language-py">The Beam SDK for Python does not currently support streaming pipelines.</span></p>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/runners/direct/index.html b/content/documentation/runners/direct/index.html
index ca4e96c..7527672 100644
--- a/content/documentation/runners/direct/index.html
+++ b/content/documentation/runners/direct/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/runners/direct/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,45 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="using-the-direct-runner">Using the Direct Runner</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Runners</span></li>
+<li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
+<li><a href="/documentation/runners/direct/">Direct Runner</a></li>
+<li><a href="/documentation/runners/apex/">Apache Apex</a></li>
+<li><a href="/documentation/runners/flink/">Apache Flink</a></li>
+<li><a href="/documentation/runners/gearpump/">Apache Gearpump</a></li>
+<li><a href="/documentation/runners/spark/">Apache Spark</a></li>
+<li><a href="/documentation/runners/dataflow/">Google Cloud Dataflow</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#direct-runner-prerequisites-and-setup">Direct Runner prerequisites and setup</a>
+ <ul>
+ <li><a href="#specify-your-dependency">Specify your dependency</a></li>
+ </ul>
+ </li>
+ <li><a href="#pipeline-options-for-the-direct-runner">Pipeline options for the Direct Runner</a></li>
+ <li><a href="#additional-information-and-caveats">Additional information and caveats</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="using-the-direct-runner">Using the Direct Runner</h1>
<nav class="language-switcher">
<strong>Adapt for:</strong>
@@ -204,7 +175,7 @@ interface for defaults and additional pipeline configuration options.</p>
<p>Local execution is limited by the memory available in your local environment. It is highly recommended that you run your pipeline with data sets small enough to fit in local memory. You can create a small in-memory data set using a <span class="language-java"><a href="/documentation/sdks/javadoc/2.1.0/index.html?org/apache/beam/sdk/transforms/Create.html"><code class="highlighter-rouge">Create</code></a></span><span class="language-py"><a href="https://github.com/apache/beam/blob/mast [...]
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/runners/flink/index.html b/content/documentation/runners/flink/index.html
index 4af3be6..1e4fb5b 100644
--- a/content/documentation/runners/flink/index.html
+++ b/content/documentation/runners/flink/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/runners/flink/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,51 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="using-the-apache-flink-runner">Using the Apache Flink Runner</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Runners</span></li>
+<li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
+<li><a href="/documentation/runners/direct/">Direct Runner</a></li>
+<li><a href="/documentation/runners/apex/">Apache Apex</a></li>
+<li><a href="/documentation/runners/flink/">Apache Flink</a></li>
+<li><a href="/documentation/runners/gearpump/">Apache Gearpump</a></li>
+<li><a href="/documentation/runners/spark/">Apache Spark</a></li>
+<li><a href="/documentation/runners/dataflow/">Google Cloud Dataflow</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#flink-runner-prerequisites-and-setup">Flink Runner prerequisites and setup</a>
+ <ul>
+ <li><a href="#specify-your-dependency">Specify your dependency</a></li>
+ </ul>
+ </li>
+ <li><a href="#executing-a-pipeline-on-a-flink-cluster">Executing a pipeline on a Flink cluster</a></li>
+ <li><a href="#pipeline-options-for-the-flink-runner">Pipeline options for the Flink Runner</a></li>
+ <li><a href="#additional-information-and-caveats">Additional information and caveats</a>
+ <ul>
+ <li><a href="#monitoring-your-job">Monitoring your job</a></li>
+ <li><a href="#streaming-execution">Streaming Execution</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="using-the-apache-flink-runner">Using the Apache Flink Runner</h1>
<nav class="language-switcher">
<strong>Adapt for:</strong>
@@ -292,7 +269,7 @@
<p>If your pipeline uses an unbounded data source or sink, the Flink Runner will automatically switch to streaming mode. You can enforce streaming mode by using the <code class="highlighter-rouge">streaming</code> setting mentioned above.</p>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/runners/gearpump/index.html b/content/documentation/runners/gearpump/index.html
index 8f33197..a6d3c68 100644
--- a/content/documentation/runners/gearpump/index.html
+++ b/content/documentation/runners/gearpump/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/runners/gearpump/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,42 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="using-the-apache-gearpump-runner">Using the Apache Gearpump Runner</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Runners</span></li>
+<li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
+<li><a href="/documentation/runners/direct/">Direct Runner</a></li>
+<li><a href="/documentation/runners/apex/">Apache Apex</a></li>
+<li><a href="/documentation/runners/flink/">Apache Flink</a></li>
+<li><a href="/documentation/runners/gearpump/">Apache Gearpump</a></li>
+<li><a href="/documentation/runners/spark/">Apache Spark</a></li>
+<li><a href="/documentation/runners/dataflow/">Google Cloud Dataflow</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#writing-beam-pipeline-with-gearpump-runner">Writing Beam Pipeline with Gearpump Runner</a></li>
+ <li><a href="#executing-the-pipeline-on-a-gearpump-cluster">Executing the pipeline on a Gearpump cluster</a></li>
+ <li><a href="#monitoring-your-application">Monitoring your application</a></li>
+ <li><a href="#pipeline-options-for-the-gearpump-runner">Pipeline options for the Gearpump Runner</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="using-the-apache-gearpump-runner">Using the Apache Gearpump Runner</h1>
<p>The Apache Gearpump Runner can be used to execute Beam pipelines using <a href="https://gearpump.apache.org">Apache Gearpump (incubating)</a>.
When you are running your pipeline with Gearpump Runner you just need to create a jar file containing your job and then it can be executed on a regular Gearpump distributed cluster, or a local cluster which is useful for development and debugging of your pipeline.</p>
@@ -270,7 +238,7 @@ And your Beam application should also pack Beam SDK explicitly and here is a sni
</tr>
</table>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/runners/jstorm/index.html b/content/documentation/runners/jstorm/index.html
index e511c8c..f6d5a70 100644
--- a/content/documentation/runners/jstorm/index.html
+++ b/content/documentation/runners/jstorm/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/runners/jstorm/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,136 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="using-the-jstorm-runner">Using the JStorm Runner</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#jstorm-runner-prerequisites-and-setup">JStorm Runner prerequisites and setup</a>
+ <ul>
+ <li><a href="#deploying-jstorm-with-your-application">Deploying JStorm with your application</a></li>
+ </ul>
+ </li>
+ <li><a href="#pipeline-options-for-the-jstorm-runner">Pipeline options for the JStorm Runner</a></li>
+ <li><a href="#additional-notes">Additional notes</a>
+ <ul>
+ <li><a href="#monitoring-your-job">Monitoring your job</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="using-the-jstorm-runner">Using the JStorm Runner</h1>
<p>The JStorm Runner can be used to execute Beam pipelines using <a href="http://jstorm.io/">JStorm</a>, while providing:</p>
@@ -243,6 +305,7 @@ When you submit a topology with argument <code class="highlighter-rouge">"--exte
<p>You can monitor your job with the JStorm UI, which displays all JStorm system metrics and Beam metrics.
For testing on local mode, you can retreive the Beam metrics with the metrics method of PipelineResult.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/runners/mapreduce/index.html b/content/documentation/runners/mapreduce/index.html
index 17c422e..c31125c 100644
--- a/content/documentation/runners/mapreduce/index.html
+++ b/content/documentation/runners/mapreduce/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/runners/mapreduce/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,41 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="using-the-apache-hadoop-mapreduce-runner">Using the Apache Hadoop MapReduce Runner</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Runners</span></li>
+<li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
+<li><a href="/documentation/runners/direct/">Direct Runner</a></li>
+<li><a href="/documentation/runners/apex/">Apache Apex</a></li>
+<li><a href="/documentation/runners/flink/">Apache Flink</a></li>
+<li><a href="/documentation/runners/gearpump/">Apache Gearpump</a></li>
+<li><a href="/documentation/runners/spark/">Apache Spark</a></li>
+<li><a href="/documentation/runners/dataflow/">Google Cloud Dataflow</a></li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#apache-hadoop-mapreduce-runner-prerequisites-and-setup">Apache Hadoop MapReduce Runner prerequisites and setup</a></li>
+ <li><a href="#deploying-apache-hadoop-mapreduce-with-your-application">Deploying Apache Hadoop MapReduce with your application</a></li>
+ <li><a href="#pipeline-options-for-the-apache-hadoop-mapreduce-runner">Pipeline options for the Apache Hadoop MapReduce Runner</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="using-the-apache-hadoop-mapreduce-runner">Using the Apache Hadoop MapReduce Runner</h1>
<p>The Apache Hadoop MapReduce Runner can be used to execute Beam pipelines using <a href="http://hadoop.apache.org/">Apache Hadoop</a>.</p>
@@ -219,6 +186,7 @@
</tr>
</table>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/runners/spark/index.html b/content/documentation/runners/spark/index.html
index c017e9a..5da5323 100644
--- a/content/documentation/runners/spark/index.html
+++ b/content/documentation/runners/spark/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/runners/spark/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,13 +77,55 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="using-the-apache-spark-runner">Using the Apache Spark Runner</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Runners</span></li>
+<li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
+<li><a href="/documentation/runners/direct/">Direct Runner</a></li>
+<li><a href="/documentation/runners/apex/">Apache Apex</a></li>
+<li><a href="/documentation/runners/flink/">Apache Flink</a></li>
+<li><a href="/documentation/runners/gearpump/">Apache Gearpump</a></li>
+<li><a href="/documentation/runners/spark/">Apache Spark</a></li>
+<li><a href="/documentation/runners/dataflow/">Google Cloud Dataflow</a></li>
+
+ </ul>
+ </nav>
+ </div>
-<p>The Apache Spark Runner can be used to execute Beam pipelines using <a href="http://spark.apache.org/">Apache Spark</a>.
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#spark-runner-prerequisites-and-setup">Spark Runner prerequisites and setup</a>
+ <ul>
+ <li><a href="#deploying-spark-with-your-application">Deploying Spark with your application</a></li>
+ <li><a href="#running-on-a-pre-deployed-spark-cluster">Running on a pre-deployed Spark cluster</a></li>
+ </ul>
+ </li>
+ <li><a href="#pipeline-options-for-the-spark-runner">Pipeline options for the Spark Runner</a></li>
+ <li><a href="#additional-notes">Additional notes</a>
+ <ul>
+ <li><a href="#using-spark-submit">Using spark-submit</a></li>
+ <li><a href="#monitoring-your-job">Monitoring your job</a></li>
+ <li><a href="#streaming-execution">Streaming Execution</a></li>
+ <li><a href="#using-a-provided-sparkcontext-and-streaminglisteners">Using a provided SparkContext and StreamingListeners</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="using-the-apache-spark-runner">Using the Apache Spark Runner</h1>
+
+<p>The Apache Spark Runner can be used to execute Beam pipelines using <a href="http://spark.apache.org/">Apache Spark</a>.
The Spark Runner can execute Spark pipelines just like a native Spark application; deploying a self-contained application for local mode, running on Spark’s Standalone RM, or using YARN or Mesos.</p>
<p>The Spark Runner executes Beam pipelines on top of Apache Spark, providing:</p>
@@ -303,6 +282,7 @@ Spark provides a <a href="http://spark.apache.org/docs/1.6.3/monitoring.html#met
<p>If you would like to execute your Spark job with a provided <code>SparkContext</code>, such as when using the <a href="https://github.com/spark-jobserver/spark-jobserver">spark-jobserver</a>, or use <code>StreamingListeners</code>, you can’t use <code>SparkPipelineOptions</code> (the context or a listener cannot be passed as a command-line argument anyway).
Instead, you should use <code>SparkContextOptions</code> which can only be used programmatically and is not a common <code>PipelineOptions</code> implementation.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/coming-soon.html b/content/documentation/sdks/feature-comparison/index.html
similarity index 50%
copy from content/coming-soon.html
copy to content/documentation/sdks/feature-comparison/index.html
index ebf9ca5..623ec8e 100644
--- a/content/coming-soon.html
+++ b/content/documentation/sdks/feature-comparison/index.html
@@ -4,7 +4,7 @@
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Apache Beam</title>
+ <title>Beam Java SDK</title>
<meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow [...]
">
<link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400" rel="stylesheet">
@@ -12,7 +12,10 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
- <link rel="canonical" href="https://beam.apache.org/coming-soon.html" data-proofer-ignore>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
+ <link rel="canonical" href="https://beam.apache.org/documentation/sdks/feature-comparison/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
<script>
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,16 +77,48 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="documentation-coming-soon">Documentation Coming Soon</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">SDKS</span></li>
+<li><a href="/documentation/sdks/java/">Java SDK</a></li>
+<li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li><a href="/documentation/sdks/python/">Python SDK</a></li>
+<li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li>
+ <span class="section-nav-list-title">DSLs</span>
-<p>You’ve reached a page that’s still in draft, or otherwise being developed! Please bear with us as we improve the documentation for Apache Beam.</p>
+ <ul class="section-nav-list">
+ <li><a href="/documentation/dsls/sql/">SQL</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
-<p><a href="/">Go Back</a> to the main Beam site.</p>
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="beam-sdk-feature-comparison">Beam SDK feature comparison</h1>
+
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/sdks/java-extensions/index.html b/content/documentation/sdks/java-extensions/index.html
index 7910cd6..c89ad06 100644
--- a/content/documentation/sdks/java-extensions/index.html
+++ b/content/documentation/sdks/java-extensions/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/sdks/java-extensions/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,13 +77,58 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-java-sdk-extensions">Apache Beam Java SDK Extensions</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">SDKS</span></li>
+<li><a href="/documentation/sdks/java/">Java SDK</a></li>
+<li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li><a href="/documentation/sdks/python/">Python SDK</a></li>
+<li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li>
+ <span class="section-nav-list-title">DSLs</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/dsls/sql/">SQL</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
-<h2 id="a-namejoin-libraryajoin-library"><a name="join-library"></a>Join-library</h2>
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#join-library">Join-library</a></li>
+ <li><a href="#sorter">Sorter</a>
+ <ul>
+ <li><a href="#caveats">Caveats</a></li>
+ <li><a href="#options">Options</a></li>
+ <li><a href="#example-usage-of-sortvalues">Example usage of <code class="highlighter-rouge">SortValues</code></a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-java-sdk-extensions">Apache Beam Java SDK Extensions</h1>
+
+<h2 id="join-library">Join-library</h2>
<p>Join-library provides inner join, outer left join, and outer right join functions. The aim
is to simplify the most common cases of join to a simple function call.</p>
@@ -170,7 +152,7 @@ PCollection<KV<String, KV<String, Long>>> joinedPcollection =
</code></pre>
</div>
-<h2 id="a-namesorterasorter"><a name="sorter"></a>Sorter</h2>
+<h2 id="sorter">Sorter</h2>
<p>This module provides the <code class="highlighter-rouge">SortValues</code> transform, which takes a <code class="highlighter-rouge">PCollection<KV<K, Iterable<KV<K2, V>>>></code> and produces a <code class="highlighter-rouge">PCollection<KV<K, Iterable<KV<K2, V>>>></code> where, for each primary key <code class="highlighter-rouge">K</code> the paired <code class="highlighter-rouge">Iterable<KV<K2, V>></code> has been sorted b [...]
@@ -202,6 +184,7 @@ PCollection<KV<String, Iterable<KV<String, Integer>>>> g
</code></pre>
</div>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/sdks/java/index.html b/content/documentation/sdks/java/index.html
index cb39f49..3a63eae 100644
--- a/content/documentation/sdks/java/index.html
+++ b/content/documentation/sdks/java/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/sdks/java/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,52 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-java-sdk">Apache Beam Java SDK</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">SDKS</span></li>
+<li><a href="/documentation/sdks/java/">Java SDK</a></li>
+<li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li><a href="/documentation/sdks/python/">Python SDK</a></li>
+<li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li>
+ <span class="section-nav-list-title">DSLs</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/dsls/sql/">SQL</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#get-started-with-the-java-sdk">Get Started with the Java SDK</a></li>
+ <li><a href="#supported-features">Supported Features</a></li>
+ <li><a href="#pipeline-io">Pipeline I/O</a></li>
+ <li><a href="#extensions">Extensions</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-java-sdk">Apache Beam Java SDK</h1>
<p>The Java SDK for Apache Beam provides a simple, powerful API for building both batch and streaming parallel data processing pipelines in Java.</p>
@@ -171,6 +149,7 @@
<li><a href="/documentation/sdks/nexmark">Nexmark</a> is a benchmark suite that runs in batch and streaming modes.</li>
</ul>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/sdks/nexmark/index.html b/content/documentation/sdks/nexmark/index.html
index 559cdc1..008e4cb 100644
--- a/content/documentation/sdks/nexmark/index.html
+++ b/content/documentation/sdks/nexmark/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/sdks/nexmark/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,165 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="nexmark-benchmark-suite">Nexmark benchmark suite</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Documentation</span></li>
+<li><a href="/documentation">Using the Documentation</a></li>
+<li><a href="/documentation/execution-model">Beam Execution Model</a></li>
+<li>
+ <span class="section-nav-list-title">Pipeline development lifecycle</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Beam programming guide</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/">Overview</a></li>
+ <li><a href="/documentation/programming-guide/#creating-a-pipeline">Pipelines</a></li>
+ <li>
+ <span class="section-nav-list-title">PCollections</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pcollections">Creating a PCollection</a></li>
+ <li><a href="/documentation/programming-guide/#pcollection-characteristics">PCollection characteristics</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#applying-transforms">Applying transforms</a></li>
+ <li>
+ <span class="section-nav-list-title">Core Beam transforms</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pardo">ParDo</a></li>
+ <li><a href="/documentation/programming-guide/#groupbykey">GroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#cogroupbykey">CoGroupByKey</a></li>
+ <li><a href="/documentation/programming-guide/#combine">Combine</a></li>
+ <li><a href="/documentation/programming-guide/#flatten">Flatten</a></li>
+ <li><a href="/documentation/programming-guide/#partition">Partition</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms">Requirements for user code</a></li>
+ <li><a href="/documentation/programming-guide/#side-inputs">Side inputs</a></li>
+ <li><a href="/documentation/programming-guide/#additional-outputs">Additional outputs</a></li>
+ <li><a href="/documentation/programming-guide/#composite-transforms">Composite transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Pipeline I/O</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#pipeline-io">Using I/O transforms</a></li>
+ <li><a href="/documentation/io/built-in/">Built-in I/O transforms</a></li>
+ <li><a href="/documentation/io/authoring-overview/">Authoring new I/O transforms</a></li>
+ <li><a href="/documentation/io/testing/">Testing I/O transforms</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Data encoding and type safety</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#data-encoding-and-type-safety">Data encoding basics</a></li>
+ <li><a href="/documentation/programming-guide/#specifying-coders">Specifying coders</a></li>
+ <li><a href="/documentation/programming-guide/#default-coders-and-the-coderregistry">Default coders and the CoderRegistry</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Windowing</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#windowing">Windowing basics</a></li>
+ <li><a href="/documentation/programming-guide/#provided-windowing-functions">Provided windowing functions</a></li>
+ <li><a href="/documentation/programming-guide/#setting-your-pcollections-windowing-function">Setting your PCollection’s windowing function</a></li>
+ <li><a href="/documentation/programming-guide/#watermarks-and-late-data">Watermarks and late data</a></li>
+ <li><a href="/documentation/programming-guide/#adding-timestamps-to-a-pcollections-elements">Adding timestamps to a PCollection’s elements</a></li>
+ </ul>
+ </li>
+ <li>
+ <span class="section-nav-list-title">Triggers</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/programming-guide/#triggers">Trigger basics</a></li>
+ <li><a href="/documentation/programming-guide/#event-time-triggers">Event time triggers and the default trigger</a></li>
+ <li><a href="/documentation/programming-guide/#processing-time-triggers">Processing time triggers</a></li>
+ <li><a href="/documentation/programming-guide/#data-driven-triggers">Data-driven triggers</a></li>
+ <li><a href="/documentation/programming-guide/#setting-a-trigger">Setting a trigger</a></li>
+ <li><a href="/documentation/programming-guide/#composite-triggers">Composite triggers</a></li>
+ </ul>
+ </li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#what-it-is">What it is</a></li>
+ <li><a href="#the-queries">The queries</a></li>
+ <li><a href="#benchmark-workload-configuration">Benchmark workload configuration</a>
+ <ul>
+ <li><a href="#events-generation-defaults">Events generation (defaults)</a></li>
+ <li><a href="#windows-defaults">Windows (defaults)</a></li>
+ <li><a href="#events-proportions-defaults">Events Proportions (defaults)</a></li>
+ <li><a href="#technical">Technical</a></li>
+ </ul>
+ </li>
+ <li><a href="#nexmark-output">Nexmark output</a></li>
+ <li><a href="#benchmark-launch-configuration">Benchmark launch configuration</a>
+ <ul>
+ <li><a href="#common-configuration-parameters">Common configuration parameters</a></li>
+ <li><a href="#available-suites">Available Suites</a></li>
+ <li><a href="#apex-runner-specific-configuration">Apex runner specific configuration</a></li>
+ <li><a href="#google-cloud-dataflow-runner-specific-configuration">Google Cloud Dataflow runner specific configuration</a></li>
+ <li><a href="#direct-runner-specific-configuration">Direct runner specific configuration</a></li>
+ <li><a href="#flink-runner-specific-configuration">Flink runner specific configuration</a></li>
+ <li><a href="#spark-runner-specific-configuration">Spark runner specific configuration</a></li>
+ </ul>
+ </li>
+ <li><a href="#current-status">Current status</a>
+ <ul>
+ <li><a href="#batch--synthetic--local">Batch / Synthetic / Local</a></li>
+ <li><a href="#streaming--synthetic--local">Streaming / Synthetic / Local</a></li>
+ <li><a href="#batch--synthetic--cluster">Batch / Synthetic / Cluster</a></li>
+ <li><a href="#streaming--synthetic--cluster">Streaming / Synthetic / Cluster</a></li>
+ </ul>
+ </li>
+ <li><a href="#running-nexmark">Running Nexmark</a>
+ <ul>
+ <li><a href="#running-smoke-suite-on-the-directrunner-local">Running SMOKE suite on the DirectRunner (local)</a></li>
+ <li><a href="#running-smoke-suite-on-the-sparkrunner-local">Running SMOKE suite on the SparkRunner (local)</a></li>
+ <li><a href="#running-smoke-suite-on-the-flinkrunner-local">Running SMOKE suite on the FlinkRunner (local)</a></li>
+ <li><a href="#running-smoke-suite-on-the-apexrunner-local">Running SMOKE suite on the ApexRunner (local)</a></li>
+ <li><a href="#running-smoke-suite-on-google-cloud-dataflow">Running SMOKE suite on Google Cloud Dataflow</a></li>
+ <li><a href="#running-query-0-on-a-spark-cluster-with-apache-hadoop-yarn">Running query 0 on a Spark cluster with Apache Hadoop YARN</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="nexmark-benchmark-suite">Nexmark benchmark suite</h1>
<h2 id="what-it-is">What it is</h2>
@@ -733,7 +824,7 @@ or may be published to Pub/Sub.</p>
</code></pre>
</div>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/sdks/python-custom-io/index.html b/content/documentation/sdks/python-custom-io/index.html
index 277ddd1..3cdea0f 100644
--- a/content/documentation/sdks/python-custom-io/index.html
+++ b/content/documentation/sdks/python-custom-io/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/sdks/python-custom-io/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,74 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="creating-new-sources-and-sinks-with-the-python-sdk">Creating New Sources and Sinks with the Python SDK</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">SDKS</span></li>
+<li><a href="/documentation/sdks/java/">Java SDK</a></li>
+<li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li><a href="/documentation/sdks/python/">Python SDK</a></li>
+<li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li>
+ <span class="section-nav-list-title">DSLs</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/dsls/sql/">SQL</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#why-create-a-new-source-or-sink">Why Create a New Source or Sink</a></li>
+ <li><a href="#basic-code-reqs">Basic Code Requirements for New Sources and Sinks</a>
+ <ul>
+ <li><a href="#serializability">Serializability</a></li>
+ <li><a href="#immutability">Immutability</a></li>
+ <li><a href="#thread-safety">Thread-Safety</a></li>
+ <li><a href="#testability">Testability</a></li>
+ </ul>
+ </li>
+ <li><a href="#creating-a-new-source">Creating a New Source</a>
+ <ul>
+ <li><a href="#implementing-the-boundedsource-subclass">Implementing the BoundedSource Subclass</a></li>
+ <li><a href="#implementing-the-rangetracker-subclass">Implementing the RangeTracker Subclass</a></li>
+ <li><a href="#convenience-source-base-classes">Convenience Source Base Classes</a></li>
+ </ul>
+ </li>
+ <li><a href="#reading-from-a-new-source">Reading from a New Source</a></li>
+ <li><a href="#creating-a-new-sink">Creating a New Sink</a>
+ <ul>
+ <li><a href="#implementing-the-sink-subclass">Implementing the Sink Subclass</a></li>
+ <li><a href="#implementing-the-writer-subclass">Implementing the Writer Subclass</a></li>
+ <li><a href="#convenience-sink-and-writer-base-classes">Convenience Sink and Writer Base Classes</a></li>
+ </ul>
+ </li>
+ <li><a href="#writing-to-a-new-sink">Writing to a New Sink</a></li>
+ <li><a href="#ptransform-wrappers">PTransform Wrappers</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="creating-new-sources-and-sinks-with-the-python-sdk">Creating New Sources and Sinks with the Python SDK</h1>
<p>The Apache Beam SDK for Python provides an extensible API that you can use to create new data sources and sinks. This tutorial shows how to create new sources and sinks using <a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/iobase.py">Beam’s Source and Sink API</a>.</p>
@@ -161,7 +161,7 @@
<p>However, if you’d like to use advanced features such as dynamic splitting and size estimation, you should use Beam’s APIs and create a new source or sink.</p>
-<h2 id="a-namebasic-code-reqsabasic-code-requirements-for-new-sources-and-sinks"><a name="basic-code-reqs"></a>Basic Code Requirements for New Sources and Sinks</h2>
+<h2 id="basic-code-reqs">Basic Code Requirements for New Sources and Sinks</h2>
<p>Services use the classes you provide to read and/or write data using multiple worker instances in parallel. As such, the code you provide for <code class="highlighter-rouge">Source</code> and <code class="highlighter-rouge">Sink</code> subclasses must meet some basic requirements:</p>
@@ -183,7 +183,7 @@
<p>You can use test harnesses and utility methods available in the <a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/source_test_utils.py">source_test_utils module</a> to develop tests for your source.</p>
-<h2 id="a-namecreating-sourcesacreating-a-new-source"><a name="creating-sources"></a>Creating a New Source</h2>
+<h2 id="creating-a-new-source">Creating a New Source</h2>
<p>You should create a new source if you’d like to use the advanced features that the Source API provides:</p>
@@ -328,7 +328,7 @@
<p>See <a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/avroio.py">AvroSource</a> for an example implementation of <code class="highlighter-rouge">FileBasedSource</code>.</p>
-<h2 id="a-namereading-sourcesareading-from-a-new-source"><a name="reading-sources"></a>Reading from a New Source</h2>
+<h2 id="reading-from-a-new-source">Reading from a New Source</h2>
<p>The following example, <code class="highlighter-rouge">CountingSource</code>, demonstrates an implementation of <code class="highlighter-rouge">BoundedSource</code> and uses the SDK-provided <code class="highlighter-rouge">RangeTracker</code> called <code class="highlighter-rouge">OffsetRangeTracker</code>.</p>
@@ -383,7 +383,7 @@
<p><strong>Note:</strong> When you create a source that end-users are going to use, it’s recommended that you do not expose the code for the source itself as demonstrated in the example above, but rather use a wrapping <code class="highlighter-rouge">PTransform</code> instead. See <a href="#ptransform-wrappers">PTransform wrappers</a> to see how and why to avoid exposing your sources.</p>
-<h2 id="a-namecreating-sinksacreating-a-new-sink"><a name="creating-sinks"></a>Creating a New Sink</h2>
+<h2 id="creating-a-new-sink">Creating a New Sink</h2>
<p>You should create a new sink if you’d like to use the advanced features that the Sink API provides, such as global initialization and finalization that allow the write operation to appear “atomic” (i.e. either all data is written or none is).</p>
@@ -465,7 +465,7 @@
<li>Setting the output MIME type</li>
</ul>
-<h2 id="a-namewriting-sinksawriting-to-a-new-sink"><a name="writing-sinks"></a>Writing to a New Sink</h2>
+<h2 id="writing-to-a-new-sink">Writing to a New Sink</h2>
<p>Consider a simple key-value storage that writes a given set of key-value pairs to a set of tables. The following is the key-value storage’s API:</p>
@@ -530,7 +530,7 @@
<p><strong>Note:</strong> When you create a sink that end-users are going to use, it’s recommended that you do not expose the code for the sink itself as demonstrated in the example above, but rather use a wrapping <code class="highlighter-rouge">PTransform</code> instead. See <a href="#ptransform-wrappers">PTransform wrappers</a> to see how and why to avoid exposing your sinks.</p>
-<h2 id="a-nameptransform-wrappersaptransform-wrappers"><a name="ptransform-wrappers"></a>PTransform Wrappers</h2>
+<h2 id="ptransform-wrappers">PTransform Wrappers</h2>
<p>If you create a new source or sink for your own use, such as for learning purposes, you should create them as explained in the sections above and use them as demonstrated in the examples.</p>
@@ -584,7 +584,7 @@ numbers = p | 'ProduceNumbers' >> ReadFromCountingSource(count)
</code></pre>
</div>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/sdks/python-pipeline-dependencies/index.html b/content/documentation/sdks/python-pipeline-dependencies/index.html
index cbc8c4a..17046d6 100644
--- a/content/documentation/sdks/python-pipeline-dependencies/index.html
+++ b/content/documentation/sdks/python-pipeline-dependencies/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/sdks/python-pipeline-dependencies/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,52 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="managing-python-pipeline-dependencies">Managing Python Pipeline Dependencies</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">SDKS</span></li>
+<li><a href="/documentation/sdks/java/">Java SDK</a></li>
+<li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li><a href="/documentation/sdks/python/">Python SDK</a></li>
+<li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li>
+ <span class="section-nav-list-title">DSLs</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/dsls/sql/">SQL</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#pypi-dependencies">PyPI Dependencies</a></li>
+ <li><a href="#local-or-nonpypi">Local or non-PyPI Dependencies</a></li>
+ <li><a href="#multiple-file-dependencies">Multiple File Dependencies</a></li>
+ <li><a href="#nonpython">Non-Python Dependencies or PyPI Dependencies with Non-Python Dependencies</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="managing-python-pipeline-dependencies">Managing Python Pipeline Dependencies</h1>
<blockquote>
<p><strong>Note:</strong> This page is only applicable to runners that do remote execution.</p>
@@ -154,7 +132,7 @@
<p><strong>Note:</strong> Remote workers used for pipeline execution typically have a standard Python 2.7 distribution installation. If your code relies only on standard Python packages, then you probably don’t need to do anything on this page.</p>
-<h2 id="a-namepypiapypi-dependencies"><a name="pypi"></a>PyPI Dependencies</h2>
+<h2 id="pypi-dependencies">PyPI Dependencies</h2>
<p>If your pipeline uses public packages from the <a href="https://pypi.python.org/pypi">Python Package Index</a>, make these packages available remotely by performing the following steps:</p>
@@ -186,7 +164,7 @@
<p><strong>Important:</strong> Remote workers will install all packages listed in the <code class="highlighter-rouge">requirements.txt</code> file. Because of this, it’s very important that you delete non-PyPI packages from the <code class="highlighter-rouge">requirements.txt</code> file, as stated in step 2. If you don’t remove non-PyPI packages, the remote workers will fail when attempting to install packages from sources that are unknown to them.</p>
-<h2 id="a-namelocalnonpypialocal-or-non-pypi-dependencies"><a name="localnonpypi"></a>Local or non-PyPI Dependencies</h2>
+<h2 id="local-or-nonpypi">Local or non-PyPI Dependencies</h2>
<p>If your pipeline uses packages that are not available publicly (e.g. packages that you’ve downloaded from a GitHub repo), make these packages available remotely by performing the following steps:</p>
@@ -218,7 +196,7 @@ package then you can build the tarball with the following command:</p>
</li>
</ol>
-<h2 id="a-namemultfilesamultiple-file-dependencies"><a name="multfiles"></a>Multiple File Dependencies</h2>
+<h2 id="multiple-file-dependencies">Multiple File Dependencies</h2>
<p>Often, your pipeline code spans multiple files. To run your project remotely, you must group these files as a Python package and specify the package when you run your pipeline. When the remote workers start, they will install your package. To group your files as a Python package and make it available remotely, perform the following steps:</p>
@@ -256,9 +234,9 @@ package then you can build the tarball with the following command:</p>
</li>
</ol>
-<p><strong>Note:</strong> If you <a href="#pypi">created a requirements.txt file</a> and your project spans multiple files, you can get rid of the <code class="highlighter-rouge">requirements.txt</code> file and instead, add all packages contained in <code class="highlighter-rouge">requirements.txt</code> to the <code class="highlighter-rouge">install_requires</code> field of the setup call (in step 1).</p>
+<p><strong>Note:</strong> If you <a href="#pypi-dependencies">created a requirements.txt file</a> and your project spans multiple files, you can get rid of the <code class="highlighter-rouge">requirements.txt</code> file and instead, add all packages contained in <code class="highlighter-rouge">requirements.txt</code> to the <code class="highlighter-rouge">install_requires</code> field of the setup call (in step 1).</p>
-<h2 id="a-namenonpythonanon-python-dependencies-or-pypi-dependencies-with-non-python-dependencies"><a name="nonpython"></a>Non-Python Dependencies or PyPI Dependencies with Non-Python Dependencies</h2>
+<h2 id="nonpython">Non-Python Dependencies or PyPI Dependencies with Non-Python Dependencies</h2>
<p>If your pipeline uses non-Python packages (e.g. packages that require installation using the <code class="highlighter-rouge">apt-get install</code> command), or uses a PyPI package that depends on non-Python dependencies during package installation, you must perform the following steps.</p>
@@ -294,7 +272,7 @@ package then you can build the tarball with the following command:</p>
<p><strong>Note:</strong> Because custom commands execute after the dependencies for your workflow are installed (by <code class="highlighter-rouge">pip</code>), you should omit the PyPI package dependency from the pipeline’s <code class="highlighter-rouge">requirements.txt</code> file and from the <code class="highlighter-rouge">install_requires</code> parameter in the <code class="highlighter-rouge">setuptools.setup()</code> call of your <code class="highlighter-rouge">setup.py</code> [...]
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/sdks/python-type-safety/index.html b/content/documentation/sdks/python-type-safety/index.html
index 8f36c49..c603651 100644
--- a/content/documentation/sdks/python-type-safety/index.html
+++ b/content/documentation/sdks/python-type-safety/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/sdks/python-type-safety/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,69 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="ensuring-python-type-safety">Ensuring Python Type Safety</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">SDKS</span></li>
+<li><a href="/documentation/sdks/java/">Java SDK</a></li>
+<li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li><a href="/documentation/sdks/python/">Python SDK</a></li>
+<li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li>
+ <span class="section-nav-list-title">DSLs</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/dsls/sql/">SQL</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#benefits-of-type-hints">Benefits of Type Hints</a></li>
+ <li><a href="#declaring-type-hints">Declaring Type Hints</a>
+ <ul>
+ <li><a href="#declaring-type-hints-inline">Declaring Type Hints Inline</a></li>
+ <li><a href="#declaring-type-hints-using-decorators">Declaring Type Hints Using Decorators</a></li>
+ <li><a href="#defining-generic-types">Defining Generic Types</a></li>
+ </ul>
+ </li>
+ <li><a href="#kinds-of-type-hints">Kinds of Type Hints</a>
+ <ul>
+ <li><a href="#simple-type-hints">Simple Type Hints</a></li>
+ <li><a href="#parameterized-type-hints">Parameterized Type Hints</a></li>
+ <li><a href="#special-type-hints">Special Type Hints</a></li>
+ </ul>
+ </li>
+ <li><a href="#runtime-type-checking">Runtime Type Checking</a></li>
+ <li><a href="#use-of-type-hints-in-coders">Use of Type Hints in Coders</a>
+ <ul>
+ <li><a href="#deterministic-coders">Deterministic Coders</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="ensuring-python-type-safety">Ensuring Python Type Safety</h1>
<p>Python is a dynamically-typed language with no static type checking. Because of the way Python’s type checking works, as well as the deferred nature of runner execution, developer productivity can easily become bottle-necked by time spent investigating type-related errors.</p>
@@ -329,7 +324,7 @@ totals = (
</code></pre>
</div>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/documentation/sdks/python/index.html b/content/documentation/sdks/python/index.html
index a7ea2ea..111cd47 100644
--- a/content/documentation/sdks/python/index.html
+++ b/content/documentation/sdks/python/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/documentation/sdks/python/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,52 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-python-sdk">Apache Beam Python SDK</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">SDKS</span></li>
+<li><a href="/documentation/sdks/java/">Java SDK</a></li>
+<li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li><a href="/documentation/sdks/python/">Python SDK</a></li>
+<li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+</li>
+<li>
+ <span class="section-nav-list-title">DSLs</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/documentation/dsls/sql/">SQL</a></li>
+ </ul>
+</li>
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#get-started-with-the-python-sdk">Get Started with the Python SDK</a></li>
+ <li><a href="#python-type-safety">Python Type Safety</a></li>
+ <li><a href="#managing-python-pipeline-dependencies">Managing Python Pipeline Dependencies</a></li>
+ <li><a href="#creating-new-sources-and-sinks">Creating New Sources and Sinks</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-python-sdk">Apache Beam Python SDK</h1>
<p>The Python SDK for Apache Beam provides a simple, powerful API for building batch data processing pipelines in Python.</p>
@@ -166,7 +144,7 @@
<p>The Beam SDK for Python provides an extensible API that you can use to create new data sources and sinks. <a href="/documentation/sdks/python-custom-io">Creating New Sources and Sinks with the Python SDK</a> shows how to create new sources and sinks using <a href="https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/iobase.py">Beam’s Source and Sink API</a>.</p>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/get-started/beam-overview/index.html b/content/get-started/beam-overview/index.html
index c84b272..8e89289 100644
--- a/content/get-started/beam-overview/index.html
+++ b/content/get-started/beam-overview/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/get-started/beam-overview/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,54 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-overview">Apache Beam Overview</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Get started</span></li>
+<li><a href="/get-started/beam-overview/">Beam Overview</a></li>
+<li>
+ <span class="section-nav-list-title">Quickstarts</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
+ <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Example Walkthroughs</span>
+ <ul class="section-nav-list">
+ <li><a href="/get-started/wordcount-example/">WordCount</a></li>
+ <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
+ </ul>
+</li>
+<li><a href="/get-started/downloads">Downloads</a></li>
+<li><a href="/get-started/support">Support</a></li>
+
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#apache-beam-sdks">Apache Beam SDKs</a></li>
+ <li><a href="#apache-beam-pipeline-runners">Apache Beam Pipeline Runners</a></li>
+ <li><a href="#get-started">Get Started</a></li>
+ <li><a href="#contribute">Contribute</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-overview">Apache Beam Overview</h1>
<p>Apache Beam is an open source, unified model for defining both batch and streaming data-parallel processing pipelines. Using one of the open source Beam SDKs, you build a program that defines the pipeline. The pipeline is then executed by one of Beam’s supported <strong>distributed processing back-ends</strong>, which include <a href="http://apex.apache.org">Apache Apex</a>, <a href="http://flink.apache.org">Apache Flink</a>, <a href="http://spark.apache.org">Apache Spark</a>, and <a [...]
@@ -197,6 +177,7 @@
<p>Beam is an <a href="http://www.apache.org">Apache Software Foundation</a> project, available under the Apache v2 license. Beam is an open source community and contributions are greatly appreciated! If you’d like to contribute, please see the <a href="/contribute/">Contribute</a> section.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/get-started/downloads/index.html b/content/get-started/downloads/index.html
index 4a3af40..0e95d22 100644
--- a/content/get-started/downloads/index.html
+++ b/content/get-started/downloads/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/get-started/downloads/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,63 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam8482-downloads">Apache Beam™ Downloads</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Get started</span></li>
+<li><a href="/get-started/beam-overview/">Beam Overview</a></li>
+<li>
+ <span class="section-nav-list-title">Quickstarts</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
+ <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Example Walkthroughs</span>
+ <ul class="section-nav-list">
+ <li><a href="/get-started/wordcount-example/">WordCount</a></li>
+ <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
+ </ul>
+</li>
+<li><a href="/get-started/downloads">Downloads</a></li>
+<li><a href="/get-started/support">Support</a></li>
+
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#api-stability">API Stability</a></li>
+ <li><a href="#releases">Releases</a>
+ <ul>
+ <li><a href="#section">2.1.0 (2017-08-23)</a></li>
+ <li><a href="#section-1">2.0.0 (2017-05-17)</a></li>
+ <li><a href="#section-2">0.6.0 (2017-03-11)</a></li>
+ <li><a href="#section-3">0.5.0 (2017-02-02)</a></li>
+ <li><a href="#section-4">0.4.0 (2016-12-29)</a></li>
+ <li><a href="#incubating-2016-10-31">0.3.0-incubating (2016-10-31)</a></li>
+ <li><a href="#incubating-2016-08-08">0.2.0-incubating (2016-08-08)</a></li>
+ <li><a href="#incubating-2016-06-15">0.1.0-incubating (2016-06-15)</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam8482-downloads">Apache Beam™ Downloads</h1>
<p>The easiest way to use Apache Beam is via one of the released versions in a central repository.
Java SDK is available on <a href="https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22">Maven Central Repository</a>,
@@ -236,6 +225,7 @@ may change at any point and are not guaranteed to remain compatible across versi
<p>The first incubating release of Apache Beam.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/get-started/index.html b/content/get-started/index.html
index d54e455..0ea5b1e 100644
--- a/content/get-started/index.html
+++ b/content/get-started/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/get-started/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,48 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="get-started-with-apache-beam">Get Started with Apache Beam</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Get started</span></li>
+<li><a href="/get-started/beam-overview/">Beam Overview</a></li>
+<li>
+ <span class="section-nav-list-title">Quickstarts</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
+ <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Example Walkthroughs</span>
+ <ul class="section-nav-list">
+ <li><a href="/get-started/wordcount-example/">WordCount</a></li>
+ <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
+ </ul>
+</li>
+<li><a href="/get-started/downloads">Downloads</a></li>
+<li><a href="/get-started/support">Support</a></li>
+
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="get-started-with-apache-beam">Get Started with Apache Beam</h1>
<p>Learn to use Beam to create data processing pipelines that run on supported processing back-ends:</p>
<h4 id="beam-overviewget-startedbeam-overview"><a href="/get-started/beam-overview">Beam Overview</a></h4>
@@ -172,6 +146,7 @@
<p>Find resources, such as mailing lists and issue tracking, to help you use Beam. Ask questions and discuss topics via <a href="http://stackoverflow.com/questions/tagged/apache-beam">Stack Overflow</a> or on Beam’s <a href="http://apachebeam.slack.com">Slack Channel</a>.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/get-started/mobile-gaming-example/index.html b/content/get-started/mobile-gaming-example/index.html
index e0f6b78..910ea45 100644
--- a/content/get-started/mobile-gaming-example/index.html
+++ b/content/get-started/mobile-gaming-example/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/get-started/mobile-gaming-example/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,72 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-mobile-gaming-pipeline-examples">Apache Beam Mobile Gaming Pipeline Examples</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Get started</span></li>
+<li><a href="/get-started/beam-overview/">Beam Overview</a></li>
+<li>
+ <span class="section-nav-list-title">Quickstarts</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
+ <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Example Walkthroughs</span>
+ <ul class="section-nav-list">
+ <li><a href="/get-started/wordcount-example/">WordCount</a></li>
+ <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
+ </ul>
+</li>
+<li><a href="/get-started/downloads">Downloads</a></li>
+<li><a href="/get-started/support">Support</a></li>
+
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#userscore-basic-score-processing-in-batch">UserScore: Basic Score Processing in Batch</a>
+ <ul>
+ <li><a href="#what-does-userscore-do">What Does UserScore Do?</a></li>
+ <li><a href="#limitations">Limitations</a></li>
+ </ul>
+ </li>
+ <li><a href="#hourlyteamscore-advanced-processing-in-batch-with-windowing">HourlyTeamScore: Advanced Processing in Batch with Windowing</a>
+ <ul>
+ <li><a href="#what-does-hourlyteamscore-do">What Does HourlyTeamScore Do?</a></li>
+ <li><a href="#limitations-1">Limitations</a></li>
+ </ul>
+ </li>
+ <li><a href="#leaderboard-streaming-processing-with-real-time-game-data">LeaderBoard: Streaming Processing with Real-Time Game Data</a>
+ <ul>
+ <li><a href="#what-does-leaderboard-do">What Does LeaderBoard Do?</a></li>
+ </ul>
+ </li>
+ <li><a href="#gamestats-abuse-detection-and-usage-analysis">GameStats: Abuse Detection and Usage Analysis</a>
+ <ul>
+ <li><a href="#what-does-gamestats-do">What Does GameStats Do?</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-mobile-gaming-pipeline-examples">Apache Beam Mobile Gaming Pipeline Examples</h1>
<ul id="markdown-toc">
<li><a href="#userscore-basic-score-processing-in-batch" id="markdown-toc-userscore-basic-score-processing-in-batch">UserScore: Basic Score Processing in Batch</a> <ul>
@@ -1070,6 +1068,7 @@
<p>We can use the resulting information to find, for example, what times of day our users are playing the longest, or which stretches of the day are more likely to see shorter play sessions.</p>
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/get-started/quickstart-java/index.html b/content/get-started/quickstart-java/index.html
index 4416237..3b1dee6 100644
--- a/content/get-started/quickstart-java/index.html
+++ b/content/get-started/quickstart-java/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/get-started/quickstart-java/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,55 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-java-sdk-quickstart">Apache Beam Java SDK Quickstart</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Get started</span></li>
+<li><a href="/get-started/beam-overview/">Beam Overview</a></li>
+<li>
+ <span class="section-nav-list-title">Quickstarts</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
+ <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Example Walkthroughs</span>
+ <ul class="section-nav-list">
+ <li><a href="/get-started/wordcount-example/">WordCount</a></li>
+ <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
+ </ul>
+</li>
+<li><a href="/get-started/downloads">Downloads</a></li>
+<li><a href="/get-started/support">Support</a></li>
+
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#set-up-your-development-environment">Set up your Development Environment</a></li>
+ <li><a href="#get-the-wordcount-code">Get the WordCount Code</a></li>
+ <li><a href="#run-wordcount">Run WordCount</a></li>
+ <li><a href="#inspect-the-results">Inspect the results</a></li>
+ <li><a href="#next-steps">Next Steps</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-java-sdk-quickstart">Apache Beam Java SDK Quickstart</h1>
<p>This Quickstart will walk you through executing your first Beam pipeline to run <a href="/get-started/wordcount-example">WordCount</a>, written using Beam’s <a href="/documentation/sdks/java">Java SDK</a>, on a <a href="/documentation#runners">runner</a> of your choice.</p>
@@ -366,7 +347,7 @@ and look through the <a href="/documentation/sdks/javadoc">Java SDK API referenc
<p>Please don’t hesitate to <a href="/get-started/support">reach out</a> if you encounter any issues!</p>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/get-started/quickstart-py/index.html b/content/get-started/quickstart-py/index.html
index 44c20e0..e913dc9 100644
--- a/content/get-started/quickstart-py/index.html
+++ b/content/get-started/quickstart-py/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/get-started/quickstart-py/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,65 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-python-sdk-quickstart">Apache Beam Python SDK Quickstart</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Get started</span></li>
+<li><a href="/get-started/beam-overview/">Beam Overview</a></li>
+<li>
+ <span class="section-nav-list-title">Quickstarts</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
+ <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Example Walkthroughs</span>
+ <ul class="section-nav-list">
+ <li><a href="/get-started/wordcount-example/">WordCount</a></li>
+ <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
+ </ul>
+</li>
+<li><a href="/get-started/downloads">Downloads</a></li>
+<li><a href="/get-started/support">Support</a></li>
+
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#set-up-your-environment">Set up your environment</a>
+ <ul>
+ <li><a href="#check-your-python-version">Check your Python version</a></li>
+ <li><a href="#install-pip">Install pip</a></li>
+ <li><a href="#install-python-virtual-environment">Install Python virtual environment</a></li>
+ </ul>
+ </li>
+ <li><a href="#get-apache-beam">Get Apache Beam</a>
+ <ul>
+ <li><a href="#create-and-activate-a-virtual-environment">Create and activate a virtual environment</a></li>
+ <li><a href="#download-and-install">Download and install</a></li>
+ </ul>
+ </li>
+ <li><a href="#execute-a-pipeline-locally">Execute a pipeline locally</a></li>
+ <li><a href="#next-steps">Next Steps</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-python-sdk-quickstart">Apache Beam Python SDK Quickstart</h1>
<p>This guide shows you how to set up your Python development environment, get the Apache Beam SDK for Python, and run an example pipeline.</p>
@@ -296,7 +287,7 @@ and look through the <a href="/documentation/sdks/pydoc">Python SDK API referenc
<p>Please don’t hesitate to <a href="/get-started/support">reach out</a> if you encounter any issues!</p>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/get-started/support/index.html b/content/get-started/support/index.html
index 5b2b9d8..3e76930 100644
--- a/content/get-started/support/index.html
+++ b/content/get-started/support/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/get-started/support/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,54 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="support">Support</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Get started</span></li>
+<li><a href="/get-started/beam-overview/">Beam Overview</a></li>
+<li>
+ <span class="section-nav-list-title">Quickstarts</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
+ <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Example Walkthroughs</span>
+ <ul class="section-nav-list">
+ <li><a href="/get-started/wordcount-example/">WordCount</a></li>
+ <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
+ </ul>
+</li>
+<li><a href="/get-started/downloads">Downloads</a></li>
+<li><a href="/get-started/support">Support</a></li>
+
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#mailing-lists">Mailing Lists</a></li>
+ <li><a href="#issue-tracking">Issue Tracking</a></li>
+ <li><a href="#beam-on-stack-overflow">Beam on Stack Overflow</a></li>
+ <li><a href="#beam-slack-channel">Beam Slack Channel</a></li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="support">Support</h1>
<p>You can find different means of support in the sections below.</p>
@@ -209,7 +189,7 @@
<p>You can participate in discussions and ask questions on the <a href="https://apachebeam.slack.com/">Beam Slack Channel</a>.</p>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/get-started/wordcount-example/index.html b/content/get-started/wordcount-example/index.html
index 76148be..469d901 100644
--- a/content/get-started/wordcount-example/index.html
+++ b/content/get-started/wordcount-example/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/get-started/wordcount-example/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -25,106 +28,40 @@
</script>
</head>
- <body class="body ">
+ <body class="body" data-spy="scroll" data-target=".page-nav" data-offset="0">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,11 +77,79 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
- <div class="body__contained">
- <h1 id="apache-beam-wordcount-examples">Apache Beam WordCount Examples</h1>
+ <div class="clearfix container-main-content">
+ <div class="section-nav closed" data-offset-top="90" data-offset-bottom="500">
+ <span class="section-nav-back glyphicon glyphicon-menu-left"></span>
+ <nav>
+ <ul class="section-nav-list" data-section-nav>
+ <li><span class="section-nav-list-main-title">Get started</span></li>
+<li><a href="/get-started/beam-overview/">Beam Overview</a></li>
+<li>
+ <span class="section-nav-list-title">Quickstarts</span>
+
+ <ul class="section-nav-list">
+ <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
+ <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
+ </ul>
+</li>
+<li>
+ <span class="section-nav-list-title">Example Walkthroughs</span>
+ <ul class="section-nav-list">
+ <li><a href="/get-started/wordcount-example/">WordCount</a></li>
+ <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
+ </ul>
+</li>
+<li><a href="/get-started/downloads">Downloads</a></li>
+<li><a href="/get-started/support">Support</a></li>
+
+
+ </ul>
+ </nav>
+ </div>
+
+ <nav class="page-nav clearfix" data-offset-top="90" data-offset-bottom="500">
+
+
+<ul class="nav">
+ <li><a href="#minimalwordcount-example">MinimalWordCount example</a>
+ <ul>
+ <li><a href="#creating-the-pipeline">Creating the pipeline</a></li>
+ <li><a href="#applying-pipeline-transforms">Applying pipeline transforms</a></li>
+ <li><a href="#running-the-pipeline">Running the pipeline</a></li>
+ </ul>
+ </li>
+ <li><a href="#wordcount-example">WordCount example</a>
+ <ul>
+ <li><a href="#specifying-explicit-dofns">Specifying explicit DoFns</a></li>
+ <li><a href="#creating-composite-transforms">Creating composite transforms</a></li>
+ <li><a href="#using-parameterizable-pipelineoptions">Using parameterizable PipelineOptions</a></li>
+ </ul>
+ </li>
+ <li><a href="#debugging-wordcount-example">Debugging WordCount example</a>
+ <ul>
+ <li><a href="#logging">Logging</a></li>
+ <li><a href="#testing-your-pipeline-via-passert">Testing your pipeline via PAssert</a></li>
+ </ul>
+ </li>
+ <li><a href="#windowedwordcount-example">WindowedWordCount example</a>
+ <ul>
+ <li><a href="#unbounded-and-bounded-pipeline-input-modes">Unbounded and bounded pipeline input modes</a></li>
+ <li><a href="#adding-timestamps-to-data">Adding timestamps to data</a></li>
+ <li><a href="#windowing">Windowing</a></li>
+ <li><a href="#reusing-ptransforms-over-windowed-pcollections">Reusing PTransforms over windowed PCollections</a></li>
+ <li><a href="#writing-results-to-an-unbounded-sink">Writing results to an unbounded sink</a></li>
+ </ul>
+ </li>
+</ul>
+
+
+ </nav>
+
+ <div class="body__contained body__section-nav">
+ <h1 id="apache-beam-wordcount-examples">Apache Beam WordCount Examples</h1>
<ul id="markdown-toc">
<li><a href="#minimalwordcount-example" id="markdown-toc-minimalwordcount-example">MinimalWordCount example</a> <ul>
@@ -1140,7 +1145,7 @@ results and writes them to a BigQuery table using <code class="highlighter-rouge
</code></pre>
</div>
-
+ </div>
</div>
<footer class="footer">
<div class="footer__contained">
diff --git a/content/index.html b/content/index.html
index 29e16f0..abc5f54 100644
--- a/content/index.html
+++ b/content/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -28,103 +31,37 @@
<body class="body body--index">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,7 +77,7 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
<div class="body__contained">
diff --git a/content/js/fix-menu.js b/content/js/fix-menu.js
new file mode 100644
index 0000000..535500a
--- /dev/null
+++ b/content/js/fix-menu.js
@@ -0,0 +1,64 @@
+$(document).ready(function() {
+ function FixMenu(conf) {
+ var idMenu = conf["classNameMenu"];
+ var CONST = {
+ DESKTOP_BREAKPOINT: 1024
+ };
+
+ return {
+ "idMenu": idMenu,
+ "menuConfig": {},
+
+ "bindEvents": function() {
+ var _self = this;
+
+ $(window).scroll(function(e) {
+ if($(window).width() > CONST.DESKTOP_BREAKPOINT) {
+ var scroll = $(window).scrollTop();
+ var scrollMenu = scroll + $("." + idMenu).height() + _self.menuConfig.offsetTop;
+ var scrollHeight = scroll + $(window).height();
+ var bodyHeightOffset = $('body')[0].scrollHeight - _self.menuConfig.offsetBottom;
+
+ if (scroll <= _self.menuConfig.offsetTop) {
+ $("." + idMenu).css({
+ top: '',
+ bottom: ''
+ });
+ } else if (scroll > _self.menuConfig.offsetTop && scrollMenu < bodyHeightOffset) {
+ $("." + idMenu).css({
+ top: _self.menuConfig.offsetTop,
+ bottom: ''
+ });
+ } else if (scrollMenu >= bodyHeightOffset) {
+ $("." + idMenu).css({
+ top: '',
+ bottom: scrollHeight - bodyHeightOffset
+ });
+ }
+ } else {
+ $("." + idMenu).css({
+ top: '',
+ bottom: ''
+ });
+ }
+ });
+
+ $(window).resize(function(e) {
+ if($(window).width() <= CONST.DESKTOP_BREAKPOINT) {
+ $("." + idMenu).css({
+ top: '',
+ bottom: ''
+ });
+ }
+ });
+ },
+ "init": function() {
+ this.menuConfig = $("." + this.idMenu).data();
+ this.bindEvents();
+ }
+ }
+ }
+
+ FixMenu({"classNameMenu":"section-nav"}).init();
+ FixMenu({ "classNameMenu":"page-nav" }).init();
+});
diff --git a/content/js/language-switch.js b/content/js/language-switch.js
index a402146..d183f81 100644
--- a/content/js/language-switch.js
+++ b/content/js/language-switch.js
@@ -80,13 +80,26 @@ $(document).ready(function() {
},
"toggle": function() {
var pref=localStorage.getItem(this.dbKey) || this.default;
+ var isPrefSelected = false;
+
// Adjusting active elements in navigation header.
$("." + this.wrapper + " li").removeClass("active").each(function() {
if ($(this).data("type") === pref) {
$(this).addClass("active");
+ isPrefSelected = true;
}
});
+ if(!isPrefSelected) {
+ pref = this.default;
+
+ $("." + this.wrapper + " li").each(function() {
+ if ($(this).data("type") === pref) {
+ $(this).addClass("active");
+ }
+ });
+ }
+
// Swapping visibility of code blocks.
$(this.selector).hide();
$("nav"+this.selector).show();
diff --git a/content/js/page-nav.js b/content/js/page-nav.js
new file mode 100644
index 0000000..a40cf0f
--- /dev/null
+++ b/content/js/page-nav.js
@@ -0,0 +1,53 @@
+$(document).ready(function() {
+ function PageNav(conf) {
+ var idPageNav = conf["classNamePageNav"];
+ var idMainContainer = conf["classNameMainContainer"];
+
+ var CONST = {
+ DESKTOP_BREAKPOINT: 1024,
+ PAGENAV_WIDTH: 240
+ };
+
+ return {
+ "idPageNav": idPageNav,
+ "idMainContainer": idMainContainer,
+
+ "setPageNav": function() {
+ var mainContainerData = {
+ width: $("." + idMainContainer).width(),
+ offset: $("." + idMainContainer).offset()
+ };
+
+ if($(window).width() > CONST.DESKTOP_BREAKPOINT) {
+ $("." + idPageNav).css({
+ left: mainContainerData.offset.left + mainContainerData.width - CONST.PAGENAV_WIDTH
+ });
+ } else {
+ $("." + idPageNav).css({
+ left: 0
+ });
+ }
+ },
+
+ "bindEvents": function() {
+ var _self = this;
+
+ $(window).resize(function() {
+ _self.setPageNav();
+ });
+ },
+
+ "init": function() {
+ this.bindEvents();
+ this.setPageNav();
+ }
+ }
+ }
+
+ PageNav(
+ {
+ "classNamePageNav":"page-nav",
+ "classNameMainContainer": "container-main-content"
+ }
+ ).init();
+});
diff --git a/content/js/section-nav.js b/content/js/section-nav.js
new file mode 100644
index 0000000..9676d74
--- /dev/null
+++ b/content/js/section-nav.js
@@ -0,0 +1,87 @@
+$(document).ready(function() {
+ function Navbar(conf) {
+ var idCTA = conf["classNameCTA"],
+ idContainer = conf["classNameContainer"],
+ idNavMask = conf["classNameMask"],
+ idBackCTA = conf["classNameBackCTA"],
+ idSectionNav = conf["classNameSectionNav"];
+
+ var CONST = {
+ OPEN_CLASS: "open",
+ DESKTOP_BREAKPOINT: 1024
+ };
+
+ return {
+ "idCTA": idCTA,
+ "idContainer": idContainer,
+ "idNavMask": idNavMask,
+ "idSectionNav": idSectionNav,
+ "idBackCTA": idBackCTA,
+ "hasSectionNav": false,
+
+ "bindEvents": function() {
+ var _self = this;
+ var sectionNavEl = $("." + idSectionNav);
+ var sectionNavHeight = $(sectionNavEl).height();
+
+ $(".container-main-content").css({"min-height": sectionNavHeight});
+
+ $(window).resize(function() {
+ if($(window).width() > CONST.DESKTOP_BREAKPOINT) {
+ var sectionNavHeight = $(sectionNavEl).height();
+ $(".container-main-content").css({"min-height": sectionNavHeight});
+ }else {
+ $(".container-main-content").css({"min-height": ''});
+ }
+ });
+
+ if(_self.hasSectionNav) {
+ $("." + _self.idCTA ).click(function(el) {
+ $("." + _self.idNavMask).addClass(CONST.OPEN_CLASS);
+ $("." + _self.idSectionNav).addClass(CONST.OPEN_CLASS);
+ });
+
+ $("." + _self.idBackCTA).click(function(el) {
+ $("." + _self.idSectionNav).removeClass(CONST.OPEN_CLASS);
+ $("." + _self.idContainer).addClass(CONST.OPEN_CLASS);
+ });
+ } else {
+ $("." + _self.idCTA ).click(function(el) {
+ $("." + _self.idNavMask).addClass(CONST.OPEN_CLASS);
+ $("." + _self.idContainer).addClass(CONST.OPEN_CLASS);
+ });
+ }
+
+ $("." + _self.idNavMask ).click(function(el) {
+ $("." + _self.idNavMask).removeClass(CONST.OPEN_CLASS);
+ $("." + _self.idContainer).removeClass(CONST.OPEN_CLASS);
+
+ if(_self.hasSectionNav) {
+ $("." + _self.idSectionNav).removeClass(CONST.OPEN_CLASS);
+ }
+ });
+ },
+ "findSectionNav": function() {
+ var sectionNavEl = $('body').find("[data-section-nav]");
+
+ if(sectionNavEl.length) {
+ this.hasSectionNav = true;
+ }
+ },
+ "init": function() {
+ this.findSectionNav();
+ this.bindEvents();
+ }
+ }
+ }
+
+ Navbar(
+ {
+ "classNameContainer":"navbar-container",
+ "classNameSectionNav": "section-nav",
+ "classNameBackCTA": "section-nav-back",
+ "classNameCTA": "navbar-toggle",
+ "classNameMask": "navbar-mask"
+ }
+ ).init();
+});
diff --git a/content/privacy_policy/index.html b/content/privacy_policy/index.html
index 47ae3dc..68a1245 100644
--- a/content/privacy_policy/index.html
+++ b/content/privacy_policy/index.html
@@ -12,6 +12,9 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/js/bootstrap.min.js"></script>
<script src="/js/language-switch.js"></script>
+ <script src="/js/fix-menu.js"></script>
+ <script src="/js/section-nav.js"></script>
+ <script src="/js/page-nav.js"></script>
<link rel="canonical" href="https://beam.apache.org/privacy_policy/" data-proofer-ignore>
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
<link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
@@ -28,103 +31,37 @@
<body class="body ">
<nav class="header navbar navbar-fixed-top">
<div class="navbar-header">
+ <button type="button" class="navbar-toggle" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
</a>
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
</div>
- <div id="navbar" class="navbar-collapse collapse">
+
+ <div class="navbar-mask closed"></div>
+
+ <div id="navbar" class="navbar-container closed">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
- <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
- <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Example Walkthroughs</li>
- <li><a href="/get-started/wordcount-example/">WordCount</a></li>
- <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Resources</li>
- <li><a href="/get-started/downloads">Downloads</a></li>
- <li><a href="/get-started/support">Support</a></li>
- </ul>
+ <li>
+ <a href="/get-started/beam-overview/">Get Started</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/documentation">Using the Documentation</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Beam Concepts</li>
- <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
- <li><a href="/documentation/execution-model/">Execution Model</a></li>
- <li><a href="/documentation/resources/">Additional Resources</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Pipeline Fundamentals</li>
- <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
- <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
- <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">SDKs</li>
- <li><a href="/documentation/sdks/java/">Java SDK</a></li>
- <li><a href="/documentation/sdks/javadoc/2.1.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li><a href="/documentation/sdks/python/">Python SDK</a></li>
- <li><a href="/documentation/sdks/pydoc/2.1.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
- width="14" height="14"
- alt="External link."></a>
- </li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Runners</li>
- <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
- <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
- <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
- <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
- <li><a href="/documentation/runners/gearpump/">Apache Gearpump Runner</a></li>
- <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
- <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
-
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">DSLs</li>
- <li><a href="/documentation/dsls/sql/">SQL</a></li>
- </ul>
+ <li>
+ <a href="/documentation/">Documentation</a>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
- <ul class="dropdown-menu">
- <li><a href="/contribute">Get Started Contributing</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Guides</li>
- <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
- <li><a href="/contribute/testing/">Testing Guide</a></li>
- <li><a href="/contribute/release-guide/">Release Guide</a></li>
- <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
- <li><a href="/contribute/runner-guide/">Runner Authoring Guide</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Technical References</li>
- <li><a href="/contribute/design-principles/">Design Principles</a></li>
- <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
- <li><a href="/contribute/source-repository/">Source Repository</a></li>
- <li><a href="/contribute/docker-images/">Docker Images</a></li>
- <li role="separator" class="divider"></li>
- <li class="dropdown-header">Promotion</li>
- <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
- <li><a href="/contribute/logos/">Logos and Design</a></li>
- <li role="separator" class="divider"></li>
- <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
- <li><a href="/contribute/team/">Team</a></li>
- </ul>
+ <li>
+ <a href="/documentation/sdks/java/">SDKS</a>
+ </li>
+ <li>
+ <a href="/documentation/runners/capability-matrix/">RUNNERS</a>
+ </li>
+ <li>
+ <a href="/contribute/">Contribute</a>
</li>
-
<li><a href="/blog">Blog</a></li>
</ul>
<ul class="nav navbar-nav navbar-right">
@@ -140,7 +77,7 @@
</ul>
</li>
</ul>
- </div><!--/.nav-collapse -->
+ </div>
</nav>
<div class="body__contained">
--
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" <co...@beam.apache.org>.