You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/05/03 21:03:13 UTC
[1/4] beam-site git commit: [BEAM-969] Add a gearpump runner web page
Repository: beam-site
Updated Branches:
refs/heads/asf-site 3cafa86a0 -> d7f468491
[BEAM-969] Add a gearpump runner web page
Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/c717c8bd
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/c717c8bd
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/c717c8bd
Branch: refs/heads/asf-site
Commit: c717c8bdc759ab8f46508ede90ed4e38424b29a7
Parents: 3cafa86
Author: huafengw <fv...@gmail.com>
Authored: Mon Apr 24 15:10:37 2017 +0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed May 3 13:59:03 2017 -0700
----------------------------------------------------------------------
src/_data/capability-matrix.yml | 124 ++++++++++++++++++++++---
src/contribute/work-in-progress.md | 2 +-
src/documentation/runners/gearpump.md | 141 +++++++++++++++++++++++++++++
src/get-started/beam-overview.md | 8 +-
src/images/logos/runners/gearpump.png | Bin 0 -> 2643 bytes
5 files changed, 261 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam-site/blob/c717c8bd/src/_data/capability-matrix.yml
----------------------------------------------------------------------
diff --git a/src/_data/capability-matrix.yml b/src/_data/capability-matrix.yml
index 3c90800..565ffba 100644
--- a/src/_data/capability-matrix.yml
+++ b/src/_data/capability-matrix.yml
@@ -9,6 +9,8 @@ columns:
name: Apache Spark
- class: apex
name: Apache Apex
+ - class: gearpump
+ name: Apache Gearpump
categories:
- description: What is being computed?
@@ -40,6 +42,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: Supported through Apex operator that wraps the function and processes data as single element bundles.
+ - class: gearpump
+ l1: 'Yes'
+ l2: fully supported
+ l3: Gearpump wraps the per-element transformation function into processor execution.
- name: GroupByKey
values:
- class: model
@@ -62,6 +68,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: "Apex runner uses the Beam code for grouping by window and thereby has support for all windowing and triggering mechanisms. Runner does not implement partitioning yet (BEAM-838)"
+ - class: gearpump
+ l1: 'Yes'
+ l2: fully supported
+ l3: "Use Gearpump's groupBy and window for key grouping and translate Beam's windowing and triggering to Gearpump's internal implementation."
- name: Flatten
values:
- class: model
@@ -84,6 +94,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: ''
+ - class: gearpump
+ l1: 'Yes'
+ l2: fully supported
+ l3: ''
- name: Combine
values:
- class: model
@@ -106,6 +120,10 @@ categories:
l1: 'Yes'
l2: 'fully supported'
l3: "Default Beam translation. Currently no efficient pre-aggregation (BEAM-935)."
+ - class: gearpump
+ l1: 'Yes'
+ l2: fully supported
+ l3: ''
- name: Composite Transforms
values:
- class: model
@@ -128,6 +146,10 @@ categories:
l1: 'Partially'
l2: supported via inlining
l3: ''
+ - class: gearpump
+ l1: 'Partially'
+ l2: supported via inlining
+ l3: ''
- name: Side Inputs
values:
- class: model
@@ -149,7 +171,11 @@ categories:
- class: apex
l1: 'Yes'
l2: size restrictions
- l3: No distributed implementation and therefore size restrictions.
+ l3: No distributed implementation and therefore size restrictions.
+ - class: gearpump
+ l1: 'Yes'
+ l2: fully supported
+ l3: Implemented by merging side input as a normal stream in Gearpump
- name: Source API
values:
- class: model
@@ -172,7 +198,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3:
-
+ - class: gearpump
+ l1: 'Yes'
+ l2: fully supported
+ l3: ''
- name: Aggregators
values:
- class: model
@@ -195,7 +224,10 @@ categories:
l1: 'No'
l2: Not implemented in runner.
l3:
-
+ - class: gearpump
+ l1: 'No'
+ l2: ''
+ l3: not implemented
- name: Stateful Processing
values:
- class: model
@@ -218,7 +250,10 @@ categories:
l1: 'No'
l2: not implemented
l3: Apex supports per-key state, so adding support for this should be easy.
-
+ - class: gearpump
+ l1: 'No'
+ l2: not implemented
+ l3: ''
- description: Where in event time?
anchor: where
color-b: '37d'
@@ -248,7 +283,10 @@ categories:
l1: 'Yes'
l2: supported
l3: ''
-
+ - class: gearpump
+ l1: 'Yes'
+ l2: supported
+ l3: ''
- name: Fixed windows
values:
- class: model
@@ -271,7 +309,10 @@ categories:
l1: 'Yes'
l2: supported
l3: ''
-
+ - class: gearpump
+ l1: 'Yes'
+ l2: supported
+ l3: ''
- name: Sliding windows
values:
- class: model
@@ -294,7 +335,10 @@ categories:
l1: 'Yes'
l2: supported
l3: ''
-
+ - class: gearpump
+ l1: 'Yes'
+ l2: supported
+ l3: ''
- name: Session windows
values:
- class: model
@@ -317,7 +361,10 @@ categories:
l1: 'Yes'
l2: supported
l3: ''
-
+ - class: gearpump
+ l1: 'Yes'
+ l2: supported
+ l3: ''
- name: Custom windows
values:
- class: model
@@ -340,7 +387,10 @@ categories:
l1: 'Yes'
l2: supported
l3: ''
-
+ - class: gearpump
+ l1: 'Yes'
+ l2: supported
+ l3: ''
- name: Custom merging windows
values:
- class: model
@@ -363,7 +413,10 @@ categories:
l1: 'Yes'
l2: supported
l3: ''
-
+ - class: gearpump
+ l1: 'Yes'
+ l2: supported
+ l3: ''
- name: Timestamp control
values:
- class: model
@@ -386,7 +439,10 @@ categories:
l1: 'Yes'
l2: supported
l3: ''
-
+ - class: gearpump
+ l1: 'Yes'
+ l2: supported
+ l3: ''
- description: When in processing time?
@@ -419,6 +475,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: ''
+ - class: gearpump
+ l1: 'No'
+ l2: ''
+ l3: ''
- name: Event-time triggers
values:
@@ -442,6 +502,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: ''
+ - class: gearpump
+ l1: 'Yes'
+ l2: fully supported
+ l3: ''
- name: Processing-time triggers
values:
@@ -465,6 +529,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: ''
+ - class: gearpump
+ l1: 'No'
+ l2: ''
+ l3: ''
- name: Count triggers
values:
@@ -488,6 +556,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: ''
+ - class: gearpump
+ l1: 'No'
+ l2: ''
+ l3: ''
- name: '[Meta]data driven triggers'
values:
@@ -511,7 +583,11 @@ categories:
- class: apex
l1: 'No'
l2: pending model support
- l3:
+ l3:
+ - class: gearpump
+ l1: 'No'
+ l2: pending model support
+ l3:
- name: Composite triggers
values:
@@ -535,6 +611,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: ''
+ - class: gearpump
+ l1: 'No'
+ l2: ''
+ l3: ''
- name: Allowed lateness
values:
@@ -558,6 +638,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: ''
+ - class: gearpump
+ l1: 'Yes'
+ l2: fully supported
+ l3: ''
- name: Timers
values:
@@ -581,6 +665,10 @@ categories:
l1: 'No'
l2: not implemented
l3: ''
+ - class: gearpump
+ l1: 'No'
+ l2: not implemented
+ l3: ''
- description: How do refinements relate?
anchor: how
@@ -612,6 +700,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: ''
+ - class: gearpump
+ l1: 'Yes'
+ l2: fully supported
+ l3: ''
- name: Accumulating
values:
@@ -635,6 +727,10 @@ categories:
l1: 'Yes'
l2: fully supported
l3: 'Size restriction, see combine support.'
+ - class: gearpump
+ l1: 'No'
+ l2: ''
+ l3: ''
- name: 'Accumulating & Retracting'
values:
@@ -659,3 +755,7 @@ categories:
l1: 'No'
l2: pending model support
l3: ''
+ - class: gearpump
+ l1: 'No'
+ l2: pending model support
+ l3: ''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam-site/blob/c717c8bd/src/contribute/work-in-progress.md
----------------------------------------------------------------------
diff --git a/src/contribute/work-in-progress.md b/src/contribute/work-in-progress.md
index c3a4d17..224dddb 100644
--- a/src/contribute/work-in-progress.md
+++ b/src/contribute/work-in-progress.md
@@ -24,7 +24,7 @@ Current branches include:
| Feature | Branch | JIRA Component | More Info |
| ---- | ---- | ---- | ---- |
-| Apache Gearpump Runner | [gearpump-runner](https://github.com/apache/beam/tree/gearpump-runner) | [runner-gearpump](https://issues.apache.org/jira/browse/BEAM/component/12330829) | [README](https://github.com/apache/beam/blob/gearpump-runner/runners/gearpump/README.md) |
+| Apache Gearpump Runner | [gearpump-runner](https://github.com/apache/beam/tree/gearpump-runner) | [runner-gearpump](https://issues.apache.org/jira/browse/BEAM/component/12330829) | [runner homepage]({{ site.baseurl }}/documentation/runners/gearpump/) |
| Apache Spark 2.0 Runner | [runners-spark2](https://github.com/apache/beam/tree/runners-spark2) | - | [thread](https://lists.apache.org/thread.html/e38ac4e4914a6cb1b865b1f32a6ca06c2be28ea4aa0f6b18393de66f@%3Cdev.beam.apache.org%3E) |
{:.table}
http://git-wip-us.apache.org/repos/asf/beam-site/blob/c717c8bd/src/documentation/runners/gearpump.md
----------------------------------------------------------------------
diff --git a/src/documentation/runners/gearpump.md b/src/documentation/runners/gearpump.md
new file mode 100644
index 0000000..7234647
--- /dev/null
+++ b/src/documentation/runners/gearpump.md
@@ -0,0 +1,141 @@
+---
+layout: default
+title: "Apache Gearpump (incubating) Runner"
+permalink: /documentation/runners/gearpump/
+---
+# Using the Apache Gearpump Runner
+
+The Apache Gearpump Runner can be used to execute Beam pipelines using [Apache Gearpump (incubating)](https://gearpump.apache.org).
+When you are running your pipeline with Gearpump Runner you just need to create a jar file containing your job and then it can be executed on a regular Gearpump distributed cluster, or a local cluster which is useful for development and debugging of your pipeline.
+
+The Gearpump Runner and Gearpump are suitable for large scale, continuous jobs, and provide:
+
+* High throughput and low latency stream processing
+* Comprehensive Dashboard for application monitoring
+* Fault-tolerance with exactly-once processing guarantees
+* Application hot re-deployment
+
+The [Beam Capability Matrix]({{ site.baseurl }}/documentation/runners/capability-matrix/) documents the currently supported capabilities of the Gearpump Runner.
+
+## Building Gearpump Runner
+Currently Gearpump Runner is on a [feature branch](https://github.com/apache/beam/tree/gearpump-runner) and in order to run your Beam pipeline with Gearpump runner, you should build out the corresponding artifacts first.
+```
+git checkout gearpump-runner
+mvn install
+```
+
+
+## Writing Beam Pipeline with Gearpump Runner
+To use the Gearpump Runner in a distributed mode, you have to setup a Gearpump cluster first by following the Gearpump [setup quickstart](http://gearpump.apache.org/releases/latest/deployment/deployment-standalone/index.html).
+
+Suppose you are writing a Beam pipeline, you can add a dependency on the latest version of the Gearpump runner by adding to your pom.xml to enable Gearpump runner.
+And your Beam application should also pack Beam SDK explicitly and here is a snippet of example pom.xml:
+```java
+<dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-gearpump</artifactId>
+ <version>{{ site.release_latest }}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.gearpump</groupId>
+ <artifactId>gearpump-streaming_2.11</artifactId>
+ <version>${gearpump.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.gearpump</groupId>
+ <artifactId>gearpump-core_2.11</artifactId>
+ <version>${gearpump.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ <version>{{ site.release_latest }}</version>
+ </dependency>
+</dependencies>
+
+<build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>shaded</shadedClassifierName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+</build>
+```
+
+After running <code>mvn package</code>, run <code>ls target</code> and you should see your application jar like:
+```
+{your_application}-{version}-shaded.jar
+```
+
+## Executing the pipeline on a Gearpump cluster
+To run against a Gearpump cluster simply run:
+```
+gear app -jar /path/to/{your_application}-{version}-shaded.jar com.beam.examples.BeamPipeline --runner=GearpumpRunner ...
+```
+
+## Monitoring your application
+You can monitor a running Gearpump application using Gearpump's Dashboard. Please follow the Gearpump [Start UI](http://gearpump.apache.org/releases/latest/deployment/deployment-standalone/index.html#start-ui) to start the dashboard.
+
+## Pipeline options for the Gearpump Runner
+
+When executing your pipeline with the Gearpump Runner, you should consider the following pipeline options.
+
+<table class="table table-bordered">
+<tr>
+ <th>Field</th>
+ <th>Description</th>
+ <th>Default Value</th>
+</tr>
+<tr>
+ <td><code>runner</code></td>
+ <td>The pipeline runner to use. This option allows you to determine the pipeline runner at runtime.</td>
+ <td>Set to <code>GearpumpRunner</code> to run using Gearpump.</td>
+</tr>
+<tr>
+ <td><code>parallelism</code></td>
+ <td>The parallelism for Gearpump processor.</td>
+ <td><code>1</code></td>
+</tr>
+<tr>
+ <td><code>applicationName</code></td>
+ <td>The application name for Gearpump runner.</td>
+ <td><code>beam_gearpump_app</code></td>
+</tr>
+</table>
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/beam-site/blob/c717c8bd/src/get-started/beam-overview.md
----------------------------------------------------------------------
diff --git a/src/get-started/beam-overview.md b/src/get-started/beam-overview.md
index 0ef5152..07b1ad9 100644
--- a/src/get-started/beam-overview.md
+++ b/src/get-started/beam-overview.md
@@ -36,10 +36,16 @@ Beam currently supports Runners that work with the following distributed process
alt="Apache Flink">
* Apache Spark <img src="{{ site.baseurl }}/images/logos/runners/spark.png"
alt="Apache Spark">
+* Apache Gearpump(incubating) <img src="{{ site.baseurl }}/images/logos/runners/gearpump.png"
+ alt="Apache Gearpump">
* Google Cloud Dataflow <img src="{{ site.baseurl }}/images/logos/runners/dataflow.png"
alt="Google Cloud Dataflow">
+
+**Note:**
-**Note:** You can always execute your pipeline locally for testing and debugging purposes.
+1. You can always execute your pipeline locally for testing and debugging purposes.
+
+2. Gearpump Runner is under development and for more details please refer to [Ongoing Projects]({{ site.baseurl }}/contribute/work-in-progress)
## Get Started
http://git-wip-us.apache.org/repos/asf/beam-site/blob/c717c8bd/src/images/logos/runners/gearpump.png
----------------------------------------------------------------------
diff --git a/src/images/logos/runners/gearpump.png b/src/images/logos/runners/gearpump.png
new file mode 100644
index 0000000..d35f44a
Binary files /dev/null and b/src/images/logos/runners/gearpump.png differ
[2/4] beam-site git commit: This closes #223: Add a gearpump runner
web page
Posted by ke...@apache.org.
This closes #223: Add a gearpump runner web page
Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/94f50512
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/94f50512
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/94f50512
Branch: refs/heads/asf-site
Commit: 94f505129854c9ccf92d418d5b1f6919e86c014f
Parents: 3cafa86 c717c8b
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed May 3 13:59:16 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed May 3 13:59:16 2017 -0700
----------------------------------------------------------------------
src/_data/capability-matrix.yml | 124 ++++++++++++++++++++++---
src/contribute/work-in-progress.md | 2 +-
src/documentation/runners/gearpump.md | 141 +++++++++++++++++++++++++++++
src/get-started/beam-overview.md | 8 +-
src/images/logos/runners/gearpump.png | Bin 0 -> 2643 bytes
5 files changed, 261 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[3/4] beam-site git commit: Minor touch-ups to Gearpump listing on
overview
Posted by ke...@apache.org.
Minor touch-ups to Gearpump listing on overview
- Alphabetical order
- Add a space
Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/da7d6c0e
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/da7d6c0e
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/da7d6c0e
Branch: refs/heads/asf-site
Commit: da7d6c0e61282fb2261131614eb633e9cafee689
Parents: 94f5051
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed May 3 13:59:48 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed May 3 13:59:48 2017 -0700
----------------------------------------------------------------------
src/get-started/beam-overview.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam-site/blob/da7d6c0e/src/get-started/beam-overview.md
----------------------------------------------------------------------
diff --git a/src/get-started/beam-overview.md b/src/get-started/beam-overview.md
index 07b1ad9..491f901 100644
--- a/src/get-started/beam-overview.md
+++ b/src/get-started/beam-overview.md
@@ -34,10 +34,10 @@ Beam currently supports Runners that work with the following distributed process
alt="Apache Apex">
* Apache Flink <img src="{{ site.baseurl }}/images/logos/runners/flink.png"
alt="Apache Flink">
+* Apache Gearpump (incubating) <img src="{{ site.baseurl }}/images/logos/runners/gearpump.png"
+ alt="Apache Gearpump">
* Apache Spark <img src="{{ site.baseurl }}/images/logos/runners/spark.png"
alt="Apache Spark">
-* Apache Gearpump(incubating) <img src="{{ site.baseurl }}/images/logos/runners/gearpump.png"
- alt="Apache Gearpump">
* Google Cloud Dataflow <img src="{{ site.baseurl }}/images/logos/runners/dataflow.png"
alt="Google Cloud Dataflow">
@@ -59,4 +59,4 @@ Get started using Beam for your data processing tasks.
## Contribute
-Beam is an [Apache Software Foundation](http://www.apache.org) project, available under the Apache v2 license. Beam is an open source community and contributions are greatly appreciated! If you'd like to contribute, please see the [Contribute]({{ site.baseurl }}/contribute/) section.
\ No newline at end of file
+Beam is an [Apache Software Foundation](http://www.apache.org) project, available under the Apache v2 license. Beam is an open source community and contributions are greatly appreciated! If you'd like to contribute, please see the [Contribute]({{ site.baseurl }}/contribute/) section.
[4/4] beam-site git commit: Regenerate website
Posted by ke...@apache.org.
Regenerate website
Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/d7f46849
Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/d7f46849
Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/d7f46849
Branch: refs/heads/asf-site
Commit: d7f46849112abafd182eb107aca486e00df07087
Parents: da7d6c0
Author: Kenneth Knowles <kl...@google.com>
Authored: Wed May 3 14:03:01 2017 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Wed May 3 14:03:01 2017 -0700
----------------------------------------------------------------------
content/contribute/work-in-progress/index.html | 2 +-
.../runners/capability-matrix/index.html | 286 +++++++++++++++++
.../documentation/runners/gearpump/index.html | 313 +++++++++++++++++++
content/get-started/beam-overview/index.html | 12 +-
content/images/logos/runners/gearpump.png | Bin 0 -> 2643 bytes
5 files changed, 611 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/contribute/work-in-progress/index.html
----------------------------------------------------------------------
diff --git a/content/contribute/work-in-progress/index.html b/content/contribute/work-in-progress/index.html
index 90eff24..07abdf1 100644
--- a/content/contribute/work-in-progress/index.html
+++ b/content/contribute/work-in-progress/index.html
@@ -184,7 +184,7 @@
<td>Apache Gearpump Runner</td>
<td><a href="https://github.com/apache/beam/tree/gearpump-runner">gearpump-runner</a></td>
<td><a href="https://issues.apache.org/jira/browse/BEAM/component/12330829">runner-gearpump</a></td>
- <td><a href="https://github.com/apache/beam/blob/gearpump-runner/runners/gearpump/README.md">README</a></td>
+ <td><a href="/documentation/runners/gearpump/">runner homepage</a></td>
</tr>
<tr>
<td>Apache Spark 2.0 Runner</td>
http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/documentation/runners/capability-matrix/index.html
----------------------------------------------------------------------
diff --git a/content/documentation/runners/capability-matrix/index.html b/content/documentation/runners/capability-matrix/index.html
index 3faf1f2..24fa451 100644
--- a/content/documentation/runners/capability-matrix/index.html
+++ b/content/documentation/runners/capability-matrix/index.html
@@ -203,6 +203,8 @@
<th class="cap-summary color-platform format-platform" style="color:#ec3">Apache Apex</th>
+ <th class="cap-summary color-platform format-platform" style="color:#ec3">Apache Gearpump</th>
+
</tr>
<tr class="cap-summary">
@@ -233,6 +235,11 @@
<td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -263,6 +270,11 @@
<td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -293,6 +305,11 @@
<td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -323,6 +340,11 @@
<td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -353,6 +375,11 @@
<td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -383,6 +410,11 @@
<td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -413,6 +445,11 @@
<td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -443,6 +480,11 @@
<td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -473,6 +515,11 @@
<td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -495,6 +542,8 @@
<th class="cap-summary color-platform format-platform" style="color:#59f">Apache Apex</th>
+ <th class="cap-summary color-platform format-platform" style="color:#59f">Apache Gearpump</th>
+
</tr>
<tr class="cap-summary">
@@ -525,6 +574,11 @@
<td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -555,6 +609,11 @@
<td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -585,6 +644,11 @@
<td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -615,6 +679,11 @@
<td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -645,6 +714,11 @@
<td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -675,6 +749,11 @@
<td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -705,6 +784,11 @@
<td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -727,6 +811,8 @@
<th class="cap-summary color-platform format-platform" style="color:#8c6">Apache Apex</th>
+ <th class="cap-summary color-platform format-platform" style="color:#8c6">Apache Gearpump</th>
+
</tr>
<tr class="cap-summary">
@@ -757,6 +843,11 @@
<td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -787,6 +878,11 @@
<td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -817,6 +913,11 @@
<td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -847,6 +948,11 @@
<td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -877,6 +983,11 @@
<td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -907,6 +1018,11 @@
<td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -937,6 +1053,11 @@
<td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -967,6 +1088,11 @@
<td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -989,6 +1115,8 @@
<th class="cap-summary color-platform format-platform" style="color:#d77">Apache Apex</th>
+ <th class="cap-summary color-platform format-platform" style="color:#d77">Apache Gearpump</th>
+
</tr>
<tr class="cap-summary">
@@ -1019,6 +1147,11 @@
<td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -1049,6 +1182,11 @@
<td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -1079,6 +1217,11 @@
<td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center>
</td>
+
+
+ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center>
+</td>
+
</tr>
<tr class="cap-summary">
@@ -1115,6 +1258,8 @@
<th class="cap color-platform format-platform" style="color:#ec3">Apache Apex</th>
+ <th class="cap color-platform format-platform" style="color:#ec3">Apache Gearpump</th>
+
</tr>
<tr class="cap">
@@ -1145,6 +1290,11 @@
<td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Supported through Apex operator that wraps the function and processes data as single element bundles.
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Gearpump wraps the per-element transformation function into processor execution.
+</td>
+
</tr>
<tr class="cap">
@@ -1175,6 +1325,11 @@
<td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Apex runner uses the Beam code for grouping by window and thereby has support for all windowing and triggering mechanisms. Runner does not implement partitioning yet (BEAM-838)
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Use Gearpump's groupBy and window for key grouping and translate Beam's windowing and triggering to Gearpump's internal implementation.
+</td>
+
</tr>
<tr class="cap">
@@ -1205,6 +1360,11 @@
<td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1235,6 +1395,11 @@
<td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Default Beam translation. Currently no efficient pre-aggregation (BEAM-935).
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1265,6 +1430,11 @@
<td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: supported via inlining</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: supported via inlining</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1295,6 +1465,11 @@
<td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: size restrictions</b></center><br />No distributed implementation and therefore size restrictions.
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Implemented by merging side input as a normal stream in Gearpump
+</td>
+
</tr>
<tr class="cap">
@@ -1325,6 +1500,11 @@
<td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1355,6 +1535,11 @@
<td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: Not implemented in runner.</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No</b></center><br />not implemented
+</td>
+
</tr>
<tr class="cap">
@@ -1385,6 +1570,11 @@
<td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: not implemented</b></center><br />Apex supports per-key state, so adding support for this should be easy.
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: not implemented</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1407,6 +1597,8 @@
<th class="cap color-platform format-platform" style="color:#59f">Apache Apex</th>
+ <th class="cap color-platform format-platform" style="color:#59f">Apache Gearpump</th>
+
</tr>
<tr class="cap">
@@ -1437,6 +1629,11 @@
<td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1467,6 +1664,11 @@
<td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1497,6 +1699,11 @@
<td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1527,6 +1734,11 @@
<td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1557,6 +1769,11 @@
<td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1587,6 +1804,11 @@
<td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1617,6 +1839,11 @@
<td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1639,6 +1866,8 @@
<th class="cap color-platform format-platform" style="color:#8c6">Apache Apex</th>
+ <th class="cap color-platform format-platform" style="color:#8c6">Apache Gearpump</th>
+
</tr>
<tr class="cap">
@@ -1669,6 +1898,11 @@
<td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1699,6 +1933,11 @@
<td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1729,6 +1968,11 @@
<td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1759,6 +2003,11 @@
<td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1789,6 +2038,11 @@
<td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1819,6 +2073,11 @@
<td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1849,6 +2108,11 @@
<td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1879,6 +2143,11 @@
<td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: not implemented</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: not implemented</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1901,6 +2170,8 @@
<th class="cap color-platform format-platform" style="color:#d77">Apache Apex</th>
+ <th class="cap color-platform format-platform" style="color:#d77">Apache Gearpump</th>
+
</tr>
<tr class="cap">
@@ -1931,6 +2202,11 @@
<td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1961,6 +2237,11 @@
<td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br />Size restriction, see combine support.
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No</b></center><br />
+</td>
+
</tr>
<tr class="cap">
@@ -1991,6 +2272,11 @@
<td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No: pending model support</b></center><br />
</td>
+
+
+ <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No: pending model support</b></center><br />
+</td>
+
</tr>
<tr class="cap">
http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/documentation/runners/gearpump/index.html
----------------------------------------------------------------------
diff --git a/content/documentation/runners/gearpump/index.html b/content/documentation/runners/gearpump/index.html
new file mode 100644
index 0000000..4a9fd82
--- /dev/null
+++ b/content/documentation/runners/gearpump/index.html
@@ -0,0 +1,313 @@
+<!DOCTYPE html>
+<html lang="en">
+
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+
+ <title>Apache Gearpump (incubating) Runner</title>
+ <meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a cloud service). Beam also brings DSL in different languages, allowing users to easily implement their data integration processes.
+">
+
+ <link rel="stylesheet" href="/styles/site.css">
+ <link rel="stylesheet" href="/css/theme.css">
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
+ <script src="/js/bootstrap.min.js"></script>
+ <script src="/js/language-switch.js"></script>
+ <link rel="canonical" href="https://beam.apache.org/documentation/runners/gearpump/" data-proofer-ignore>
+ <link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml">
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-73650088-1', 'auto');
+ ga('send', 'pageview');
+
+ </script>
+ <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
+</head>
+
+
+ <body role="document">
+
+ <nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <div class="navbar-header">
+ <a href="/" class="navbar-brand" >
+ <img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png">
+ </a>
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ </div>
+ <div id="navbar" class="navbar-collapse collapse">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/get-started/beam-overview/">Beam Overview</a></li>
+ <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li>
+ <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header">Example Walkthroughs</li>
+ <li><a href="/get-started/wordcount-example/">WordCount</a></li>
+ <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header">Resources</li>
+ <li><a href="/get-started/downloads">Downloads</a></li>
+ <li><a href="/get-started/support">Support</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/documentation">Using the Documentation</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header">Beam Concepts</li>
+ <li><a href="/documentation/programming-guide/">Programming Guide</a></li>
+ <li><a href="/documentation/resources/">Additional Resources</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header">Pipeline Fundamentals</li>
+ <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li>
+ <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li>
+ <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header">SDKs</li>
+ <li><a href="/documentation/sdks/java/">Java SDK</a></li>
+ <li><a href="/documentation/sdks/javadoc/0.6.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+ </li>
+ <li><a href="/documentation/sdks/python/">Python SDK</a></li>
+ <li><a href="/documentation/sdks/pydoc/0.6.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png"
+ width="14" height="14"
+ alt="External link."></a>
+ </li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header">Runners</li>
+ <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li>
+ <li><a href="/documentation/runners/direct/">Direct Runner</a></li>
+ <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li>
+ <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li>
+ <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li>
+ <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/contribute">Get Started Contributing</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header">Guides</li>
+ <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li>
+ <li><a href="/contribute/testing/">Testing Guide</a></li>
+ <li><a href="/contribute/release-guide/">Release Guide</a></li>
+ <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header">Technical References</li>
+ <li><a href="/contribute/design-principles/">Design Principles</a></li>
+ <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li>
+ <li><a href="/contribute/source-repository/">Source Repository</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header">Promotion</li>
+ <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li>
+ <li><a href="/contribute/logos/">Logos and Design</a></li>
+ <li role="separator" class="divider"></li>
+ <li><a href="/contribute/maturity-model/">Maturity Model</a></li>
+ <li><a href="/contribute/team/">Team</a></li>
+ </ul>
+ </li>
+
+ <li><a href="/blog">Blog</a></li>
+ </ul>
+ <ul class="nav navbar-nav navbar-right">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false"><img src="https://www.apache.org/foundation/press/kit/feather_small.png" alt="Apache Logo" style="height:24px;">Apache Software Foundation<span class="caret"></span></a>
+ <ul class="dropdown-menu dropdown-menu-right">
+ <li><a href="http://www.apache.org/">ASF Homepage</a></li>
+ <li><a href="http://www.apache.org/licenses/">License</a></li>
+ <li><a href="http://www.apache.org/security/">Security</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
+ <li><a href="https://www.apache.org/foundation/policies/conduct">Code of Conduct</a></li>
+ </ul>
+ </li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+</nav>
+
+
+<link rel="stylesheet" href="">
+
+
+ <div class="container" role="main">
+
+ <div class="row">
+ <h1 id="using-the-apache-gearpump-runner">Using the Apache Gearpump Runner</h1>
+
+<p>The Apache Gearpump Runner can be used to execute Beam pipelines using <a href="https://gearpump.apache.org">Apache Gearpump (incubating)</a>.
+When you are running your pipeline with Gearpump Runner you just need to create a jar file containing your job and then it can be executed on a regular Gearpump distributed cluster, or a local cluster which is useful for development and debugging of your pipeline.</p>
+
+<p>The Gearpump Runner and Gearpump are suitable for large scale, continuous jobs, and provide:</p>
+
+<ul>
+ <li>High throughput and low latency stream processing</li>
+ <li>Comprehensive Dashboard for application monitoring</li>
+ <li>Fault-tolerance with exactly-once processing guarantees</li>
+ <li>Application hot re-deployment</li>
+</ul>
+
+<p>The <a href="/documentation/runners/capability-matrix/">Beam Capability Matrix</a> documents the currently supported capabilities of the Gearpump Runner.</p>
+
+<h2 id="building-gearpump-runner">Building Gearpump Runner</h2>
+<p>Currently Gearpump Runner is on a <a href="https://github.com/apache/beam/tree/gearpump-runner">feature branch</a> and in order to run your Beam pipeline with Gearpump runner, you should build out the corresponding artifacts first.</p>
+<div class="highlighter-rouge"><pre class="highlight"><code>git checkout gearpump-runner
+mvn install
+</code></pre>
+</div>
+
+<h2 id="writing-beam-pipeline-with-gearpump-runner">Writing Beam Pipeline with Gearpump Runner</h2>
+<p>To use the Gearpump Runner in a distributed mode, you have to setup a Gearpump cluster first by following the Gearpump <a href="http://gearpump.apache.org/releases/latest/deployment/deployment-standalone/index.html">setup quickstart</a>.</p>
+
+<p>Suppose you are writing a Beam pipeline, you can add a dependency on the latest version of the Gearpump runner by adding to your pom.xml to enable Gearpump runner.
+And your Beam application should also pack Beam SDK explicitly and here is a snippet of example pom.xml:</p>
+<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="o"><</span><span class="n">dependencies</span><span class="o">></span>
+ <span class="o"><</span><span class="n">dependency</span><span class="o">></span>
+ <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">beam</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">beam</span><span class="o">-</span><span class="n">runners</span><span class="o">-</span><span class="n">gearpump</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="mf">0.6</span><span class="o">.</span><span class="mi">0</span><span class="o"></</span><span class="n">version</span><span class="o">></span>
+ <span class="o"></</span><span class="n">dependency</span><span class="o">></span>
+
+ <span class="o"><</span><span class="n">dependency</span><span class="o">></span>
+ <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">gearpump</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">gearpump</span><span class="o">-</span><span class="n">streaming_2</span><span class="o">.</span><span class="mi">11</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="err">$</span><span class="o">{</span><span class="n">gearpump</span><span class="o">.</span><span class="na">version</span><span class="o">}</</span><span class="n">version</span><span class="o">></span>
+ <span class="o"><</span><span class="n">scope</span><span class="o">></span><span class="n">provided</span><span class="o"></</span><span class="n">scope</span><span class="o">></span>
+ <span class="o"></</span><span class="n">dependency</span><span class="o">></span>
+
+ <span class="o"><</span><span class="n">dependency</span><span class="o">></span>
+ <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">gearpump</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">gearpump</span><span class="o">-</span><span class="n">core_2</span><span class="o">.</span><span class="mi">11</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="err">$</span><span class="o">{</span><span class="n">gearpump</span><span class="o">.</span><span class="na">version</span><span class="o">}</</span><span class="n">version</span><span class="o">></span>
+ <span class="o"><</span><span class="n">scope</span><span class="o">></span><span class="n">provided</span><span class="o"></</span><span class="n">scope</span><span class="o">></span>
+ <span class="o"></</span><span class="n">dependency</span><span class="o">></span>
+
+ <span class="o"><</span><span class="n">dependency</span><span class="o">></span>
+ <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">beam</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">beam</span><span class="o">-</span><span class="n">sdks</span><span class="o">-</span><span class="n">java</span><span class="o">-</span><span class="n">core</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="mf">0.6</span><span class="o">.</span><span class="mi">0</span><span class="o"></</span><span class="n">version</span><span class="o">></span>
+ <span class="o"></</span><span class="n">dependency</span><span class="o">></span>
+<span class="o"></</span><span class="n">dependencies</span><span class="o">></span>
+
+<span class="o"><</span><span class="n">build</span><span class="o">></span>
+ <span class="o"><</span><span class="n">plugins</span><span class="o">></span>
+ <span class="o"><</span><span class="n">plugin</span><span class="o">></span>
+ <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">maven</span><span class="o">.</span><span class="na">plugins</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">maven</span><span class="o">-</span><span class="n">shade</span><span class="o">-</span><span class="n">plugin</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span>
+ <span class="o"><</span><span class="n">configuration</span><span class="o">></span>
+ <span class="o"><</span><span class="n">createDependencyReducedPom</span><span class="o">></span><span class="kc">false</span><span class="o"></</span><span class="n">createDependencyReducedPom</span><span class="o">></span>
+ <span class="o"><</span><span class="n">filters</span><span class="o">></span>
+ <span class="o"><</span><span class="n">filter</span><span class="o">></span>
+ <span class="o"><</span><span class="n">artifact</span><span class="o">>*:*</</span><span class="n">artifact</span><span class="o">></span>
+ <span class="o"><</span><span class="n">excludes</span><span class="o">></span>
+ <span class="o"><</span><span class="n">exclude</span><span class="o">></span><span class="n">META</span><span class="o">-</span><span class="n">INF</span><span class="o">/*.</span><span class="na">SF</span><span class="o"></</span><span class="n">exclude</span><span class="o">></span>
+ <span class="o"><</span><span class="n">exclude</span><span class="o">></span><span class="n">META</span><span class="o">-</span><span class="n">INF</span><span class="o">/*.</span><span class="na">DSA</span><span class="o"></</span><span class="n">exclude</span><span class="o">></span>
+ <span class="o"><</span><span class="n">exclude</span><span class="o">></span><span class="n">META</span><span class="o">-</span><span class="n">INF</span><span class="o">/*.</span><span class="na">RSA</span><span class="o"></</span><span class="n">exclude</span><span class="o">></span>
+ <span class="o"></</span><span class="n">excludes</span><span class="o">></span>
+ <span class="o"></</span><span class="n">filter</span><span class="o">></span>
+ <span class="o"></</span><span class="n">filters</span><span class="o">></span>
+ <span class="o"></</span><span class="n">configuration</span><span class="o">></span>
+ <span class="o"><</span><span class="n">executions</span><span class="o">></span>
+ <span class="o"><</span><span class="n">execution</span><span class="o">></span>
+ <span class="o"><</span><span class="n">phase</span><span class="o">></span><span class="kn">package</span><span class="o"></</span><span class="n">phase</span><span class="o">></span>
+ <span class="o"><</span><span class="n">goals</span><span class="o">></span>
+ <span class="o"><</span><span class="n">goal</span><span class="o">></span><span class="n">shade</span><span class="o"></</span><span class="n">goal</span><span class="o">></span>
+ <span class="o"></</span><span class="n">goals</span><span class="o">></span>
+ <span class="o"><</span><span class="n">configuration</span><span class="o">></span>
+ <span class="o"><</span><span class="n">shadedArtifactAttached</span><span class="o">></span><span class="kc">true</span><span class="o"></</span><span class="n">shadedArtifactAttached</span><span class="o">></span>
+ <span class="o"><</span><span class="n">shadedClassifierName</span><span class="o">></span><span class="n">shaded</span><span class="o"></</span><span class="n">shadedClassifierName</span><span class="o">></span>
+ <span class="o"></</span><span class="n">configuration</span><span class="o">></span>
+ <span class="o"></</span><span class="n">execution</span><span class="o">></span>
+ <span class="o"></</span><span class="n">executions</span><span class="o">></span>
+ <span class="o"></</span><span class="n">plugin</span><span class="o">></span>
+ <span class="o"></</span><span class="n">plugins</span><span class="o">></span>
+<span class="o"></</span><span class="n">build</span><span class="o">></span>
+</code></pre>
+</div>
+
+<p>After running <code>mvn package</code>, run <code>ls target</code> and you should see your application jar like:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="err">your_application</span><span class="p">}</span><span class="err">-</span><span class="p">{</span><span class="err">version</span><span class="p">}</span><span class="err">-shaded.jar</span><span class="w">
+</span></code></pre>
+</div>
+
+<h2 id="executing-the-pipeline-on-a-gearpump-cluster">Executing the pipeline on a Gearpump cluster</h2>
+<p>To run against a Gearpump cluster simply run:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code>gear app -jar /path/to/{your_application}-{version}-shaded.jar com.beam.examples.BeamPipeline --runner=GearpumpRunner ...
+</code></pre>
+</div>
+
+<h2 id="monitoring-your-application">Monitoring your application</h2>
+<p>You can monitor a running Gearpump application using Gearpump’s Dashboard. Please follow the Gearpump <a href="http://gearpump.apache.org/releases/latest/deployment/deployment-standalone/index.html#start-ui">Start UI</a> to start the dashboard.</p>
+
+<h2 id="pipeline-options-for-the-gearpump-runner">Pipeline options for the Gearpump Runner</h2>
+
+<p>When executing your pipeline with the Gearpump Runner, you should consider the following pipeline options.</p>
+
+<table class="table table-bordered">
+<tr>
+ <th>Field</th>
+ <th>Description</th>
+ <th>Default Value</th>
+</tr>
+<tr>
+ <td><code>runner</code></td>
+ <td>The pipeline runner to use. This option allows you to determine the pipeline runner at runtime.</td>
+ <td>Set to <code>GearpumpRunner</code> to run using Gearpump.</td>
+</tr>
+<tr>
+ <td><code>parallelism</code></td>
+ <td>The parallelism for Gearpump processor.</td>
+ <td><code>1</code></td>
+</tr>
+<tr>
+ <td><code>applicationName</code></td>
+ <td>The application name for Gearpump runner.</td>
+ <td><code>beam_gearpump_app</code></td>
+</tr>
+</table>
+
+
+ </div>
+
+
+ <hr>
+ <div class="row">
+ <div class="col-xs-12">
+ <footer>
+ <p class="text-center">
+ © Copyright
+ <a href="http://www.apache.org">The Apache Software Foundation</a>,
+ 2017. All Rights Reserved.
+ </p>
+ <p class="text-center">
+ <a href="/privacy_policy">Privacy Policy</a> |
+ <a href="/feed.xml">RSS Feed</a>
+ </p>
+ </footer>
+ </div>
+ </div>
+ <!-- container div end -->
+</div>
+
+
+ </body>
+
+</html>
http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/get-started/beam-overview/index.html
----------------------------------------------------------------------
diff --git a/content/get-started/beam-overview/index.html b/content/get-started/beam-overview/index.html
index c284b15..6bae7d3 100644
--- a/content/get-started/beam-overview/index.html
+++ b/content/get-started/beam-overview/index.html
@@ -177,11 +177,21 @@
<ul>
<li>Apache Apex <img src="/images/logos/runners/apex.png" alt="Apache Apex" /></li>
<li>Apache Flink <img src="/images/logos/runners/flink.png" alt="Apache Flink" /></li>
+ <li>Apache Gearpump (incubating) <img src="/images/logos/runners/gearpump.png" alt="Apache Gearpump" /></li>
<li>Apache Spark <img src="/images/logos/runners/spark.png" alt="Apache Spark" /></li>
<li>Google Cloud Dataflow <img src="/images/logos/runners/dataflow.png" alt="Google Cloud Dataflow" /></li>
</ul>
-<p><strong>Note:</strong> You can always execute your pipeline locally for testing and debugging purposes.</p>
+<p><strong>Note:</strong></p>
+
+<ol>
+ <li>
+ <p>You can always execute your pipeline locally for testing and debugging purposes.</p>
+ </li>
+ <li>
+ <p>Gearpump Runner is under development and for more details please refer to <a href="/contribute/work-in-progress">Ongoing Projects</a></p>
+ </li>
+</ol>
<h2 id="get-started">Get Started</h2>
http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/images/logos/runners/gearpump.png
----------------------------------------------------------------------
diff --git a/content/images/logos/runners/gearpump.png b/content/images/logos/runners/gearpump.png
new file mode 100644
index 0000000..d35f44a
Binary files /dev/null and b/content/images/logos/runners/gearpump.png differ