You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by wh...@apache.org on 2016/01/12 23:13:19 UTC
[01/10] storm git commit: STORM-1366. Add documentation for StormSQL
into the website.
Repository: storm
Updated Branches:
refs/heads/asf-site b962d32e6 -> 573bdcf1d
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/storm-solr.html
----------------------------------------------------------------------
diff --git a/_site/documentation/storm-solr.html b/_site/documentation/storm-solr.html
index 5304340..6d6418b 100644
--- a/_site/documentation/storm-solr.html
+++ b/_site/documentation/storm-solr.html
@@ -104,28 +104,28 @@ responsible for making the update calls to Solr, thus updating the index of the
describe in detail the two key components of the Storm Solr integration, the <code>SolrUpdateBolt</code>, and the <code>Mappers</code>, <code>SolrFieldsMapper</code>, and <code>SolrJsonMapper</code>.</p>
<h2 id="storm-bolt-with-json-mapper-and-count-based-commit-strategy">Storm Bolt With JSON Mapper and Count Based Commit Strategy</h2>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="k">new</span> <span class="nf">SolrUpdateBolt</span><span class="o">(</span><span class="n">solrConfig</span><span class="o">,</span> <span class="n">solrMapper</span><span class="o">,</span> <span class="n">solrCommitStgy</span><span class="o">)</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="k">new</span> <span class="nf">SolrUpdateBolt</span><span class="p">(</span><span class="n">solrConfig</span><span class="o">,</span> <span class="n">solrMapper</span><span class="o">,</span> <span class="n">solrCommitStgy</span><span class="o">)</span>
- <span class="c1">// zkHostString for Solr 'gettingstarted' example</span>
- <span class="n">SolrConfig</span> <span class="n">solrConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SolrConfig</span><span class="o">(</span><span class="s">"127.0.0.1:9983"</span><span class="o">);</span>
+ <span class="c1">// zkHostString for Solr 'gettingstarted' example</span>
+ <span class="n">SolrConfig</span> <span class="n">solrConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SolrConfig</span><span class="o">(</span><span class="s">"127.0.0.1:9983"</span><span class="o">);</span>
- <span class="c1">// JSON Mapper used to generate 'SolrRequest' requests to update the "gettingstarted" Solr collection with JSON content declared the tuple field with name "JSON"</span>
- <span class="n">SolrMapper</span> <span class="n">solrMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SolrJsonMapper</span><span class="o">.</span><span class="na">Builder</span><span class="o">(</span><span class="s">"gettingstarted"</span><span class="o">,</span> <span class="s">"JSON"</span><span class="o">).</span><span class="na">build</span><span class="o">();</span>
+ <span class="c1">// JSON Mapper used to generate 'SolrRequest' requests to update the "gettingstarted" Solr collection with JSON content declared the tuple field with name "JSON"</span>
+ <span class="n">SolrMapper</span> <span class="n">solrMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SolrJsonMapper</span><span class="o">.</span><span class="na">Builder</span><span class="o">(</span><span class="s">"gettingstarted"</span><span class="o">,</span> <span class="s">"JSON"</span><span class="o">).</span><span class="na">build</span><span class="o">();</span>
<span class="c1">// Acks every other five tuples. Setting to null acks every tuple</span>
- <span class="n">SolrCommitStrategy</span> <span class="n">solrCommitStgy</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">CountBasedCommit</span><span class="o">(</span><span class="mi">5</span><span class="o">);</span>
+ <span class="n">SolrCommitStrategy</span> <span class="n">solrCommitStgy</span> <span class="o">=</span> <span class="k">new</span> <span class="n">CountBasedCommit</span><span class="o">(</span><span class="mi">5</span><span class="o">);</span>
</code></pre></div>
<h2 id="trident-topology-with-fields-mapper">Trident Topology With Fields Mapper</h2>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="k">new</span> <span class="nf">SolrStateFactory</span><span class="o">(</span><span class="n">solrConfig</span><span class="o">,</span> <span class="n">solrMapper</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="k">new</span> <span class="nf">SolrStateFactory</span><span class="p">(</span><span class="n">solrConfig</span><span class="o">,</span> <span class="n">solrMapper</span><span class="o">);</span>
- <span class="c1">// zkHostString for Solr 'gettingstarted' example</span>
- <span class="n">SolrConfig</span> <span class="n">solrConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SolrConfig</span><span class="o">(</span><span class="s">"127.0.0.1:9983"</span><span class="o">);</span>
+ <span class="c1">// zkHostString for Solr 'gettingstarted' example</span>
+ <span class="n">SolrConfig</span> <span class="n">solrConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SolrConfig</span><span class="o">(</span><span class="s">"127.0.0.1:9983"</span><span class="o">);</span>
- <span class="cm">/* Solr Fields Mapper used to generate 'SolrRequest' requests to update the "gettingstarted" Solr collection. The Solr index is updated using the field values of the tuple fields that match static or dynamic fields declared in the schema object build using schemaBuilder */</span>
- <span class="n">SolrMapper</span> <span class="n">solrMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SolrFieldsMapper</span><span class="o">.</span><span class="na">Builder</span><span class="o">(</span><span class="n">schemaBuilder</span><span class="o">,</span> <span class="s">"gettingstarted"</span><span class="o">).</span><span class="na">build</span><span class="o">();</span>
+ <span class="cm">/* Solr Fields Mapper used to generate 'SolrRequest' requests to update the "gettingstarted" Solr collection. The Solr index is updated using the field values of the tuple fields that match static or dynamic fields declared in the schema object build using schemaBuilder */</span>
+ <span class="n">SolrMapper</span> <span class="n">solrMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SolrFieldsMapper</span><span class="o">.</span><span class="na">Builder</span><span class="o">(</span><span class="n">schemaBuilder</span><span class="o">,</span> <span class="s">"gettingstarted"</span><span class="o">).</span><span class="na">build</span><span class="o">();</span>
<span class="c1">// builds the Schema object from the JSON representation of the schema as returned by the URL http://localhost:8983/solr/gettingstarted/schema/ </span>
- <span class="n">SchemaBuilder</span> <span class="n">schemaBuilder</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">RestJsonSchemaBuilder</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">,</span> <span class="s">"8983"</span><span class="o">,</span> <span class="s">"gettingstarted"</span><span class="o">)</span>
+ <span class="n">SchemaBuilder</span> <span class="n">schemaBuilder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">RestJsonSchemaBuilder</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">,</span> <span class="s">"8983"</span><span class="o">,</span> <span class="s">"gettingstarted"</span><span class="o">)</span>
</code></pre></div>
<h2 id="solrupdatebolt">SolrUpdateBolt</h2>
@@ -179,8 +179,8 @@ that is part of the <code>SolrFieldsMapper.Builder</code> builder class. </p>
field separates each value with the token % instead of the default | . To use the default token you can ommit the call to the method
<code>setMultiValueFieldToken</code>.</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="k">new</span> <span class="n">SolrFieldsMapper</span><span class="o">.</span><span class="na">Builder</span><span class="o">(</span>
- <span class="k">new</span> <span class="nf">RestJsonSchemaBuilder</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">,</span> <span class="s">"8983"</span><span class="o">,</span> <span class="s">"gettingstarted"</span><span class="o">),</span> <span class="s">"gettingstarted"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">setMultiValueFieldToken</span><span class="o">(</span><span class="s">"%"</span><span class="o">).</span><span class="na">build</span><span class="o">();</span>
+ <span class="k">new</span> <span class="n">RestJsonSchemaBuilder</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">,</span> <span class="s">"8983"</span><span class="o">,</span> <span class="s">"gettingstarted"</span><span class="o">),</span> <span class="s">"gettingstarted"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">setMultiValueFieldToken</span><span class="o">(</span><span class="s">"%"</span><span class="o">).</span><span class="na">build</span><span class="o">();</span>
</code></pre></div>
<h1 id="build-and-run-bundled-examples">Build And Run Bundled Examples</h1>
@@ -194,7 +194,7 @@ and then generate an uber jar with all the dependencies.</p>
<h2 id="use-the-maven-shade-plugin-to-build-the-uber-jar">Use the Maven Shade Plugin to Build the Uber Jar</h2>
<p>Add the following to <code>REPO_HOME/storm/external/storm-solr/pom.xml</code></p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"> <plugin>
+<div class="highlight"><pre><code class="language-" data-lang=""> <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.1</version>
@@ -206,7 +206,7 @@ and then generate an uber jar with all the dependencies.</p>
</goals>
<configuration>
<transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.apache.storm.solr.topology.SolrJsonTopology</mainClass>
</transformer>
</transformers>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/feed.xml
----------------------------------------------------------------------
diff --git a/_site/feed.xml b/_site/feed.xml
index f0ebac6..75e99d4 100644
--- a/_site/feed.xml
+++ b/_site/feed.xml
@@ -5,9 +5,9 @@
<description></description>
<link>http://storm.apache.org/</link>
<atom:link href="http://storm.apache.org/feed.xml" rel="self" type="application/rss+xml"/>
- <pubDate>Thu, 05 Nov 2015 14:02:33 -0500</pubDate>
- <lastBuildDate>Thu, 05 Nov 2015 14:02:33 -0500</lastBuildDate>
- <generator>Jekyll v2.5.3</generator>
+ <pubDate>Wed, 06 Jan 2016 14:21:20 -0800</pubDate>
+ <lastBuildDate>Wed, 06 Jan 2016 14:21:20 -0800</lastBuildDate>
+ <generator>Jekyll v3.0.1</generator>
<item>
<title>Storm 0.9.6 released</title>
@@ -34,7 +34,7 @@
<li>STORM-933: NullPointerException during KafkaSpout deactivation</li>
</ul>
</description>
- <pubDate>Thu, 05 Nov 2015 00:00:00 -0500</pubDate>
+ <pubDate>Thu, 05 Nov 2015 00:00:00 -0800</pubDate>
<link>http://storm.apache.org/2015/11/05/storm096-released.html</link>
<guid isPermaLink="true">http://storm.apache.org/2015/11/05/storm096-released.html</guid>
@@ -91,7 +91,7 @@
<li>STORM-992: A bug in the timer.clj might cause unexpected delay to schedule new event</li>
</ul>
</description>
- <pubDate>Thu, 05 Nov 2015 00:00:00 -0500</pubDate>
+ <pubDate>Thu, 05 Nov 2015 00:00:00 -0800</pubDate>
<link>http://storm.apache.org/2015/11/05/storm0100-released.html</link>
<guid isPermaLink="true">http://storm.apache.org/2015/11/05/storm0100-released.html</guid>
@@ -104,7 +104,7 @@
<p>Aside from many stability and performance improvements, this release includes a number of important new features, some of which are highlighted below.</p>
-<h2 id="secure,-multi-tenant-deployment">Secure, Multi-Tenant Deployment</h2>
+<h2 id="secure-multi-tenant-deployment">Secure, Multi-Tenant Deployment</h2>
<p>Much like the early days of Hadoop, Apache Storm originally evolved in an environment where security was not a high-priority concern. Rather, it was assumed that Storm would be deployed to environments suitably cordoned off from security threats. While a large number of users were comfortable setting up their own security measures for Storm (usually at the Firewall/OS level), this proved a hindrance to broader adoption among larger enterprises where security policies prohibited deployment without specific safeguards.</p>
@@ -192,7 +192,7 @@ that allows users to stream data from Storm directly into hive. Storm&#39;s
<p>Further information can be found in the <a href="https://github.com/apache/storm/blob/v0.10.0-beta/external/storm-redis/README.md">storm-redis documentation</a>.</p>
-<h2 id="jdbc/rdbms-integration">JDBC/RDBMS Integration</h2>
+<h2 id="jdbc-rdbms-integration">JDBC/RDBMS Integration</h2>
<p>Many stream processing data flows require accessing data from or writing data to a relational data store. Storm 0.10.0 introduces highly flexible and customizable support for integrating with virtually any JDBC-compliant database.</p>
@@ -385,7 +385,7 @@ that allows users to stream data from Storm directly into hive. Storm&#39;s
<li>STORM-130: Supervisor getting killed due to java.io.FileNotFoundException: File &#39;../stormconf.ser&#39; does not exist.</li>
</ul>
</description>
- <pubDate>Mon, 15 Jun 2015 00:00:00 -0400</pubDate>
+ <pubDate>Mon, 15 Jun 2015 00:00:00 -0700</pubDate>
<link>http://storm.apache.org/2015/06/15/storm0100-beta-released.html</link>
<guid isPermaLink="true">http://storm.apache.org/2015/06/15/storm0100-beta-released.html</guid>
@@ -411,7 +411,7 @@ that allows users to stream data from Storm directly into hive. Storm&#39;s
<li>STORM-130: Supervisor getting killed due to java.io.FileNotFoundException: File &#39;../stormconf.ser&#39; does not exist.</li>
</ul>
</description>
- <pubDate>Thu, 04 Jun 2015 00:00:00 -0400</pubDate>
+ <pubDate>Thu, 04 Jun 2015 00:00:00 -0700</pubDate>
<link>http://storm.apache.org/2015/06/04/storm095-released.html</link>
<guid isPermaLink="true">http://storm.apache.org/2015/06/04/storm095-released.html</guid>
@@ -438,7 +438,7 @@ that allows users to stream data from Storm directly into hive. Storm&#39;s
<li>STORM-130: Supervisor getting killed due to java.io.FileNotFoundException: File &#39;../stormconf.ser&#39; does not exist.</li>
</ul>
</description>
- <pubDate>Wed, 25 Mar 2015 00:00:00 -0400</pubDate>
+ <pubDate>Wed, 25 Mar 2015 00:00:00 -0700</pubDate>
<link>http://storm.apache.org/2015/03/25/storm094-released.html</link>
<guid isPermaLink="true">http://storm.apache.org/2015/03/25/storm094-released.html</guid>
@@ -634,7 +634,7 @@ that allows users to stream data from Storm directly into hive. Storm&#39;s
<li>STORM-514: Update storm-starter README now that Storm has graduated from Incubator</li>
</ul>
</description>
- <pubDate>Tue, 25 Nov 2014 00:00:00 -0500</pubDate>
+ <pubDate>Tue, 25 Nov 2014 00:00:00 -0800</pubDate>
<link>http://storm.apache.org/2014/11/25/storm093-released.html</link>
<guid isPermaLink="true">http://storm.apache.org/2014/11/25/storm093-released.html</guid>
@@ -649,7 +649,7 @@ that allows users to stream data from Storm directly into hive. Storm&#39;s
<p>We heartily encourage you to <a href="http://storm.apache.org/downloads.html">test the 0.9.3 release candidate</a> and provide your feedback regarding any issues via <a href="http://storm.apache.org/community.html">our mailing lists</a>, which is an easy and valuable way to contribute back to the Storm community and to help us moving to an official 0.9.3 release. You can find the <a href="http://storm.apache.org/downloads.html">0.9.3 release candidate in our Downloads section</a>.</p>
</description>
- <pubDate>Mon, 20 Oct 2014 00:00:00 -0400</pubDate>
+ <pubDate>Mon, 20 Oct 2014 00:00:00 -0700</pubDate>
<link>http://storm.apache.org/2014/10/20/storm093-release-candidate.html</link>
<guid isPermaLink="true">http://storm.apache.org/2014/10/20/storm093-release-candidate.html</guid>
@@ -693,7 +693,7 @@ that allows users to stream data from Storm directly into hive. Storm&#39;s
<p>The <code>storm-kafka</code> module can be found in the <code>/external/</code> directory of the source tree and binary distributions. The <code>external</code> area has been set up to contain projects that while not required by Storm, are often used in conjunction with Storm to integrate with some other technology. Such projects also come with a maintenance committment from at least one Storm committer to ensure compatibility with Storm&#39;s main codebase as it evolves.</p>
<p>The <code>storm-kafka</code> dependency is available now from Maven Central at the following coordinates:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">groupId: org.apache.storm
+<div class="highlight"><pre><code class="language-" data-lang="">groupId: org.apache.storm
artifactId: storm-kafka
version: 0.9.2-incubating
</code></pre></div>
@@ -784,7 +784,7 @@ version: 0.9.2-incubating
<li>STORM-146: Unit test regression when storm is compiled with 3.4.5 zookeeper</li>
</ul>
</description>
- <pubDate>Wed, 25 Jun 2014 00:00:00 -0400</pubDate>
+ <pubDate>Wed, 25 Jun 2014 00:00:00 -0700</pubDate>
<link>http://storm.apache.org/2014/06/25/storm092-released.html</link>
<guid isPermaLink="true">http://storm.apache.org/2014/06/25/storm092-released.html</guid>
@@ -831,7 +831,7 @@ version: 0.9.2-incubating
</tr>
</tbody></table>
</description>
- <pubDate>Tue, 17 Jun 2014 00:00:00 -0400</pubDate>
+ <pubDate>Tue, 17 Jun 2014 00:00:00 -0700</pubDate>
<link>http://storm.apache.org/2014/06/17/contest-results.html</link>
<guid isPermaLink="true">http://storm.apache.org/2014/06/17/contest-results.html</guid>
@@ -909,7 +909,7 @@ version: 0.9.2-incubating
<p>The authors of the selected Apache Storm logo(s) will be required to donate them to the Apache Storm project and complete and <a href="http://www.apache.org/licenses/icla.txt">Apache Individual Contributor License Agreement(ICLA)</a></p>
</description>
- <pubDate>Thu, 10 Apr 2014 00:00:00 -0400</pubDate>
+ <pubDate>Thu, 10 Apr 2014 00:00:00 -0700</pubDate>
<link>http://storm.apache.org/2014/04/10/storm-logo-contest.html</link>
<guid isPermaLink="true">http://storm.apache.org/2014/04/10/storm-logo-contest.html</guid>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/getting-help.html
----------------------------------------------------------------------
diff --git a/_site/getting-help.html b/_site/getting-help.html
index 198849b..0734b28 100644
--- a/_site/getting-help.html
+++ b/_site/getting-help.html
@@ -104,13 +104,13 @@
<p>You can view the archives of the mailing list <a href="http://mail-archives.apache.org/mod_mbox/storm-dev/">here</a>.</p>
-<h4 id="which-list-should-i-send/subscribe-to?">Which list should I send/subscribe to?</h4>
+<h4 id="which-list-should-i-send-subscribe-to">Which list should I send/subscribe to?</h4>
<p>If you are using a pre-built binary distribution of Storm, then chances are you should send questions, comments, storm-related announcements, etc. to <a href="user@storm.apache.org">user@storm.apache.org</a>. </p>
<p>If you are building storm from source, developing new features, or otherwise hacking storm source code, then <a href="dev@storm.apache.org">dev@storm.apache.org</a> is more appropriate. </p>
-<h4 id="what-will-happen-with-storm-user@googlegroups.com?">What will happen with <a href="mailto:storm-user@googlegroups.com">storm-user@googlegroups.com</a>?</h4>
+<h4 id="what-will-happen-with-storm-user-googlegroups-com">What will happen with <a href="mailto:storm-user@googlegroups.com">storm-user@googlegroups.com</a>?</h4>
<p>All existing messages will remain archived there, and can be accessed/searched <a href="https://groups.google.com/forum/#!forum/storm-user">here</a>.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/tutorial.html
----------------------------------------------------------------------
diff --git a/_site/tutorial.html b/_site/tutorial.html
index 76a5cf5..559ea39 100644
--- a/_site/tutorial.html
+++ b/_site/tutorial.html
@@ -113,7 +113,7 @@
<p>To do realtime computation on Storm, you create what are called "topologies". A topology is a graph of computation. Each node in a topology contains processing logic, and links between nodes indicate how data should be passed around between nodes.</p>
<p>Running a topology is straightforward. First, you package all your code and dependencies into a single jar. Then, you run a command like the following:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">storm jar all-my-code.jar backtype.storm.MyTopology arg1 arg2
+<div class="highlight"><pre><code class="language-" data-lang="">storm jar all-my-code.jar backtype.storm.MyTopology arg1 arg2
</code></pre></div>
<p>This runs the class <code>backtype.storm.MyTopology</code> with the arguments <code>arg1</code> and <code>arg2</code>. The main function of the class defines the topology and submits it to Nimbus. The <code>storm jar</code> part takes care of connecting to Nimbus and uploading the jar.</p>
@@ -148,20 +148,20 @@
<span class="kd">private</span> <span class="n">OutputCollectorBase</span> <span class="n">_collector</span><span class="o">;</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">OutputCollectorBase</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">OutputCollectorBase</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_collector</span> <span class="o">=</span> <span class="n">collector</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">input</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">input</span><span class="o">)</span> <span class="o">{</span>
<span class="kt">int</span> <span class="n">val</span> <span class="o">=</span> <span class="n">input</span><span class="o">.</span><span class="na">getInteger</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span>
- <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">input</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">val</span><span class="o">*</span><span class="mi">2</span><span class="o">,</span> <span class="n">val</span><span class="o">*</span><span class="mi">3</span><span class="o">));</span>
+ <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">input</span><span class="o">,</span> <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">val</span><span class="o">*</span><span class="mi">2</span><span class="o">,</span> <span class="n">val</span><span class="o">*</span><span class="mi">3</span><span class="o">));</span>
<span class="n">_collector</span><span class="o">.</span><span class="na">ack</span><span class="o">(</span><span class="n">input</span><span class="o">);</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"double"</span><span class="o">,</span> <span class="s">"triple"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"double"</span><span class="o">,</span> <span class="s">"triple"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
@@ -170,12 +170,12 @@
<h2 id="a-simple-topology">A simple topology</h2>
<p>Let's take a look at a simple topology to explore the concepts more and see how the code shapes up. Let's look at the <code>ExclamationTopology</code> definition from storm-starter:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TopologyBuilder</span><span class="o">();</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"words"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">TestWordSpout</span><span class="o">(),</span> <span class="mi">10</span><span class="o">);</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"exclaim1"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">ExclamationBolt</span><span class="o">(),</span> <span class="mi">3</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"words"</span><span class="o">);</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"exclaim2"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">ExclamationBolt</span><span class="o">(),</span> <span class="mi">2</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"exclaim1"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TopologyBuilder</span><span class="o">();</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"words"</span><span class="o">,</span> <span class="k">new</span> <span class="n">TestWordSpout</span><span class="o">(),</span> <span class="mi">10</span><span class="o">);</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"exclaim1"</span><span class="o">,</span> <span class="k">new</span> <span class="n">ExclamationBolt</span><span class="o">(),</span> <span class="mi">3</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"words"</span><span class="o">);</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"exclaim2"</span><span class="o">,</span> <span class="k">new</span> <span class="n">ExclamationBolt</span><span class="o">(),</span> <span class="mi">2</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"exclaim1"</span><span class="o">);</span>
</code></pre></div>
<p>This topology contains a spout and two bolts. The spout emits words, and each bolt appends the string "!!!" to its input. The nodes are arranged in a line: the spout emits to the first bolt which then emits to the second bolt. If the spout emits the tuples ["bob"] and ["john"], then the second bolt will emit the words ["bob!!!!!!"] and ["john!!!!!!"].</p>
@@ -188,19 +188,19 @@
<p><code>setBolt</code> returns an <a href="/javadoc/apidocs/backtype/storm/topology/InputDeclarer.html">InputDeclarer</a> object that is used to define the inputs to the Bolt. Here, component "exclaim1" declares that it wants to read all the tuples emitted by component "words" using a shuffle grouping, and component "exclaim2" declares that it wants to read all the tuples emitted by component "exclaim1" using a shuffle grouping. "shuffle grouping" means that tuples should be randomly distributed from the input tasks to the bolt's tasks. There are many ways to group data between components. These will be explained in a few sections.</p>
<p>If you wanted component "exclaim2" to read all the tuples emitted by both component "words" and component "exclaim1", you would write component "exclaim2"'s definition like this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"exclaim2"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">ExclamationBolt</span><span class="o">(),</span> <span class="mi">5</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"words"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"exclaim1"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"exclaim2"</span><span class="o">,</span> <span class="k">new</span> <span class="n">ExclamationBolt</span><span class="o">(),</span> <span class="mi">5</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"words"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"exclaim1"</span><span class="o">);</span>
</code></pre></div>
<p>As you can see, input declarations can be chained to specify multiple sources for the Bolt.</p>
<p>Let's dig into the implementations of the spouts and bolts in this topology. Spouts are responsible for emitting new messages into the topology. <code>TestWordSpout</code> in this topology emits a random word from the list ["nathan", "mike", "jackson", "golda", "bertels"] as a 1-tuple every 100ms. The implementation of <code>nextTuple()</code> in TestWordSpout looks like this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kt">void</span> <span class="nf">nextTuple</span><span class="o">()</span> <span class="o">{</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kt">void</span> <span class="nf">nextTuple</span><span class="p">(</span><span class="o">)</span> <span class="o">{</span>
<span class="n">Utils</span><span class="o">.</span><span class="na">sleep</span><span class="o">(</span><span class="mi">100</span><span class="o">);</span>
- <span class="kd">final</span> <span class="n">String</span><span class="o">[]</span> <span class="n">words</span> <span class="o">=</span> <span class="k">new</span> <span class="n">String</span><span class="o">[]</span> <span class="o">{</span><span class="s">"nathan"</span><span class="o">,</span> <span class="s">"mike"</span><span class="o">,</span> <span class="s">"jackson"</span><span class="o">,</span> <span class="s">"golda"</span><span class="o">,</span> <span class="s">"bertels"</span><span class="o">};</span>
- <span class="kd">final</span> <span class="n">Random</span> <span class="n">rand</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Random</span><span class="o">();</span>
+ <span class="kd">final</span> <span class="n">String</span><span class="o">[]</span> <span class="n">words</span> <span class="o">=</span> <span class="k">new</span> <span class="n">String</span><span class="o">[]</span> <span class="o">{</span><span class="s">"nathan"</span><span class="o">,</span> <span class="s">"mike"</span><span class="o">,</span> <span class="s">"jackson"</span><span class="o">,</span> <span class="s">"golda"</span><span class="o">,</span> <span class="s">"bertels"</span><span class="o">};</span>
+ <span class="kd">final</span> <span class="n">Random</span> <span class="n">rand</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Random</span><span class="o">();</span>
<span class="kd">final</span> <span class="n">String</span> <span class="n">word</span> <span class="o">=</span> <span class="n">words</span><span class="o">[</span><span class="n">rand</span><span class="o">.</span><span class="na">nextInt</span><span class="o">(</span><span class="n">words</span><span class="o">.</span><span class="na">length</span><span class="o">)];</span>
- <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
+ <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
<span class="o">}</span>
</code></pre></div>
<p>As you can see, the implementation is very straightforward.</p>
@@ -210,27 +210,27 @@
<span class="n">OutputCollector</span> <span class="n">_collector</span><span class="o">;</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">OutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">OutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_collector</span> <span class="o">=</span> <span class="n">collector</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">tuple</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> <span class="o">+</span> <span class="s">"!!!"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">tuple</span><span class="o">,</span> <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> <span class="o">+</span> <span class="s">"!!!"</span><span class="o">));</span>
<span class="n">_collector</span><span class="o">.</span><span class="na">ack</span><span class="o">(</span><span class="n">tuple</span><span class="o">);</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">cleanup</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">cleanup</span><span class="o">()</span> <span class="o">{</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="n">Map</span> <span class="nf">getComponentConfiguration</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">Map</span> <span class="n">getComponentConfiguration</span><span class="o">()</span> <span class="o">{</span>
<span class="k">return</span> <span class="kc">null</span><span class="o">;</span>
<span class="o">}</span>
<span class="o">}</span>
@@ -252,19 +252,19 @@
<span class="n">OutputCollector</span> <span class="n">_collector</span><span class="o">;</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">OutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">OutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_collector</span> <span class="o">=</span> <span class="n">collector</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">tuple</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> <span class="o">+</span> <span class="s">"!!!"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">tuple</span><span class="o">,</span> <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> <span class="o">+</span> <span class="s">"!!!"</span><span class="o">));</span>
<span class="n">_collector</span><span class="o">.</span><span class="na">ack</span><span class="o">(</span><span class="n">tuple</span><span class="o">);</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
@@ -277,14 +277,14 @@
<p>In distributed mode, Storm operates as a cluster of machines. When you submit a topology to the master, you also submit all the code necessary to run the topology. The master will take care of distributing your code and allocating workers to run your topology. If workers go down, the master will reassign them somewhere else. You can read more about running topologies on a cluster on <a href="/documentation/Running-topologies-on-a-production-cluster.html">Running topologies on a production cluster</a>. </p>
<p>Here's the code that runs <code>ExclamationTopology</code> in local mode:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Config</span> <span class="n">conf</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Config</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Config</span> <span class="n">conf</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Config</span><span class="o">();</span>
<span class="n">conf</span><span class="o">.</span><span class="na">setDebug</span><span class="o">(</span><span class="kc">true</span><span class="o">);</span>
<span class="n">conf</span><span class="o">.</span><span class="na">setNumWorkers</span><span class="o">(</span><span class="mi">2</span><span class="o">);</span>
-<span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">LocalCluster</span><span class="o">();</span>
-<span class="n">cluster</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"test"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createTopology</span><span class="o">());</span>
+<span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LocalCluster</span><span class="o">();</span>
+<span class="n">cluster</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"test"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createTopology</span><span class="o">());</span>
<span class="n">Utils</span><span class="o">.</span><span class="na">sleep</span><span class="o">(</span><span class="mi">10000</span><span class="o">);</span>
-<span class="n">cluster</span><span class="o">.</span><span class="na">killTopology</span><span class="o">(</span><span class="s">"test"</span><span class="o">);</span>
+<span class="n">cluster</span><span class="o">.</span><span class="na">killTopology</span><span class="o">(</span><span class="s">"test"</span><span class="o">);</span>
<span class="n">cluster</span><span class="o">.</span><span class="na">shutdown</span><span class="o">();</span>
</code></pre></div>
<p>First, the code defines an in-process cluster by creating a <code>LocalCluster</code> object. Submitting topologies to this virtual cluster is identical to submitting topologies to distributed clusters. It submits a topology to the <code>LocalCluster</code> by calling <code>submitTopology</code>, which takes as arguments a name for the running topology, a configuration for the topology, and then the topology itself.</p>
@@ -311,13 +311,13 @@
<p>When a task for Bolt A emits a tuple to Bolt B, which task should it send the tuple to?</p>
<p>A "stream grouping" answers this question by telling Storm how to send tuples between sets of tasks. Before we dig into the different kinds of stream groupings, let's take a look at another topology from <a href="http://github.com/apache/storm/blob/master/examples/storm-starter">storm-starter</a>. This <a href="https://github.com/apache/storm/blob/master/examples/storm-starter/src/jvm/storm/starter/WordCountTopology.java">WordCountTopology</a> reads sentences off of a spout and streams out of <code>WordCountBolt</code> the total number of times it has seen that word before:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TopologyBuilder</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TopologyBuilder</span><span class="o">();</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">RandomSentenceSpout</span><span class="o">(),</span> <span class="mi">5</span><span class="o">);</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">SplitSentence</span><span class="o">(),</span> <span class="mi">8</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">);</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"count"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">WordCount</span><span class="o">(),</span> <span class="mi">12</span><span class="o">)</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">,</span> <span class="k">new</span> <span class="n">RandomSentenceSpout</span><span class="o">(),</span> <span class="mi">5</span><span class="o">);</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="n">SplitSentence</span><span class="o">(),</span> <span class="mi">8</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">);</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"count"</span><span class="o">,</span> <span class="k">new</span> <span class="n">WordCount</span><span class="o">(),</span> <span class="mi">12</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
</code></pre></div>
<p><code>SplitSentence</code> emits a tuple for each word in each sentence it receives, and <code>WordCount</code> keeps a map in memory from word to count. Each time <code>WordCount</code> receives a word, it updates its state and emits the new word count.</p>
@@ -337,12 +337,12 @@
<p>Here's the definition of the <code>SplitSentence</code> bolt from <code>WordCountTopology</code>:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">SplitSentence</span> <span class="kd">extends</span> <span class="n">ShellBolt</span> <span class="kd">implements</span> <span class="n">IRichBolt</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="nf">SplitSentence</span><span class="o">()</span> <span class="o">{</span>
- <span class="kd">super</span><span class="o">(</span><span class="s">"python"</span><span class="o">,</span> <span class="s">"splitsentence.py"</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="n">SplitSentence</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">super</span><span class="o">(</span><span class="s">"python"</span><span class="o">,</span> <span class="s">"splitsentence.py"</span><span class="o">);</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
@@ -351,7 +351,7 @@
<span class="k">class</span> <span class="nc">SplitSentenceBolt</span><span class="p">(</span><span class="n">storm</span><span class="o">.</span><span class="n">BasicBolt</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">process</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tup</span><span class="p">):</span>
- <span class="n">words</span> <span class="o">=</span> <span class="n">tup</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">" "</span><span class="p">)</span>
+ <span class="n">words</span> <span class="o">=</span> <span class="n">tup</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">" "</span><span class="p">)</span>
<span class="k">for</span> <span class="n">word</span> <span class="ow">in</span> <span class="n">words</span><span class="p">:</span>
<span class="n">storm</span><span class="o">.</span><span class="n">emit</span><span class="p">([</span><span class="n">word</span><span class="p">])</span>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/documentation.html
----------------------------------------------------------------------
diff --git a/documentation.html b/documentation.html
index 348edd8..a2fc2f9 100644
--- a/documentation.html
+++ b/documentation.html
@@ -49,8 +49,8 @@ title: Documentation
</ul>
</div>
<div role="tabpanel" class="tab-pane" id="integration">
- <p>The following modules are included in the Apache Storm distribution and are not required for storm to operate,
- but are useful for extending Storm in order to provide additional functionality such as integration with other
+ <p>The following modules are included in the Apache Storm distribution and are not required for storm to operate,
+ but are useful for extending Storm in order to provide additional functionality such as integration with other
technologies frequently used in combination with Storm.</p>
<ul>
<li><a href="documentation/storm-kafka.html">Kafka</a></li>
@@ -62,6 +62,7 @@ title: Documentation
<li><a href="documentation/storm-solr.html">Solr</a></li>
<li><a href="documentation/storm-eventhubs.html">Azure EventHubs</a></li>
<li><a href="documentation/flux.html">Flux</a> (declarative wiring/configuration of Topologies)</li>
+ <li><a href="documentation/storm-sql.html">SQL</a> (writing topologies in SQL)</li>
</ul>
</div>
<div role="tabpanel" class="tab-pane" id="intermediate">
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/documentation/images/storm-sql-internal-example.png
----------------------------------------------------------------------
diff --git a/documentation/images/storm-sql-internal-example.png b/documentation/images/storm-sql-internal-example.png
new file mode 100644
index 0000000..74828d5
Binary files /dev/null and b/documentation/images/storm-sql-internal-example.png differ
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/documentation/images/storm-sql-internal-workflow.png
----------------------------------------------------------------------
diff --git a/documentation/images/storm-sql-internal-workflow.png b/documentation/images/storm-sql-internal-workflow.png
new file mode 100644
index 0000000..655c1c4
Binary files /dev/null and b/documentation/images/storm-sql-internal-workflow.png differ
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/documentation/storm-sql-internal.md
----------------------------------------------------------------------
diff --git a/documentation/storm-sql-internal.md b/documentation/storm-sql-internal.md
new file mode 100644
index 0000000..05ca1a9
--- /dev/null
+++ b/documentation/storm-sql-internal.md
@@ -0,0 +1,55 @@
+---
+title: The Internals of Storm SQL
+layout: documentation
+documentation: true
+---
+
+This page describes the design and the implementation of the Storm SQL integration.
+
+## Overview
+
+SQL is a well-adopted yet complicated standard. Several projects including Drill, Hive, Phoenix and Spark have invested significantly in their SQL layers. One of the main design goal of StormSQL is to leverage the existing investments for these projects. StormSQL leverages [Apache Calcite](///calcite.apache.org) to implement the SQL standard. StormSQL focuses on compiling the SQL statements to Storm / Trident topologies so that they can be executed in Storm clusters.
+
+Figure 1 describes the workflow of executing a SQL query in StormSQL. First, users provide a sequence of SQL statements. StormSQL parses the SQL statements and translates them to a Calcite logical plan. A logical plan consists of a sequence of SQL logical operators that describe how the query should be executed irrespective to the underlying execution engines. Some examples of logical operators include `TableScan`, `Filter`, `Projection` and `GroupBy`.
+
+<div align="center">
+<img title="Workflow of StormSQL" src="images/storm-sql-internal-workflow.png" style="max-width: 80rem"/>
+
+<p>Figure 1: Workflow of StormSQL.</p>
+</div>
+
+The next step is to compile the logical execution plan down to a physical execution plan. A physical plan consists of physical operators that describes how to execute the SQL query in *StormSQL*. Physical operators such as `Filter`, `Projection`, and `GroupBy` are directly mapped to operations in Trident topologies. StormSQL also compiles expressions in the SQL statements into Java byte codes and plugs them into the Trident topologies.
+
+Finally, StormSQL packages both the Java byte codes and the topology into a JAR and submits it to the Storm cluster. Storm schedules and executes the JAR in the same way of it executes other Storm topologies.
+
+The follow code blocks show an example query that filters and projects results from a Kafka stream.
+
+```
+CREATE EXTERNAL TABLE ORDERS (ID INT PRIMARY KEY, UNIT_PRICE INT, QUANTITY INT) LOCATION 'kafka://localhost:2181/brokers?topic=orders' ...
+
+CREATE EXTERNAL TABLE LARGE_ORDERS (ID INT PRIMARY KEY, TOTAL INT) LOCATION 'kafka://localhost:2181/brokers?topic=large_orders' ...
+
+INSERT INTO LARGE_ORDERS SELECT ID, UNIT_PRICE * QUANTITY AS TOTAL FROM ORDERS WHERE UNIT_PRICE * QUANTITY > 50
+```
+
+The first two SQL statements define the inputs and outputs of external data. Figure 2 describes the processes of how StormSQL takes the last `SELECT` query and compiles it down to Trident topology.
+
+<div align="center">
+<img title="Compiling the example query to Trident topology" src="images/storm-sql-internal-example.png" style="max-width: 80rem"/>
+
+<p>Figure 2: Compiling the example query to Trident topology.</p>
+</div>
+
+
+## Constraints of querying streaming tables
+
+There are several constraints when querying tables that represent a real-time data stream:
+
+* The `ORDER BY` clause cannot be applied to a stream.
+* There is at least one monotonic field in the `GROUP BY` clauses to allow StormSQL bounds the size of the buffer.
+
+For more information please refer to https://calcite.incubator.apache.org/docs/stream.html.
+
+## Dependency
+
+StormSQL does not ship the dependency of the external data sources in the packaged JAR. The users have to provide the dependency in the `extlib` directory of the worker node.
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/documentation/storm-sql.md
----------------------------------------------------------------------
diff --git a/documentation/storm-sql.md b/documentation/storm-sql.md
new file mode 100644
index 0000000..72dc776
--- /dev/null
+++ b/documentation/storm-sql.md
@@ -0,0 +1,97 @@
+---
+title: Storm SQL integration
+layout: documentation
+documentation: true
+---
+
+The Storm SQL integration allows users to run SQL queries over streaming data in Storm. Not only the SQL interface allows faster development cycles on streaming analytics, but also opens up the opportunities to unify batch data processing like [Apache Hive](///hive.apache.org) and real-time streaming data analytics.
+
+At a very high level StormSQL compiles the SQL queries to [Trident](Trident-API-Overview.html) topologies and executes them in Storm clusters. This document provides information of how to use StormSQL as end users. For people that are interested in more details in the design and the implementation of StormSQL please refer to the [this](storm-sql-internal.html) page.
+
+## Usage
+
+Run the ``storm sql`` command to compile SQL statements into Trident topology, and submit it to the Storm cluster
+
+```
+$ bin/storm sql <sql-file> <topo-name>
+```
+
+In which `sql-file` contains a list of SQL statements to be executed, and `topo-name` is the name of the topology.
+
+
+## Supported Features
+
+The following features are supported in the current repository:
+
+* Streaming from and to external data sources
+* Filtering tuples
+* Projections
+
+## Specifying External Data Sources
+
+In StormSQL data is represented by external tables. Users can specify data sources using the `CREATE EXTERNAL TABLE` statement. The syntax of `CREATE EXTERNAL TABLE` closely follows the one defined in [Hive Data Definition Language](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL):
+
+```
+CREATE EXTERNAL TABLE table_name field_list
+ [ STORED AS
+ INPUTFORMAT input_format_classname
+ OUTPUTFORMAT output_format_classname
+ ]
+ LOCATION location
+ [ TBLPROPERTIES tbl_properties ]
+ [ AS select_stmt ]
+```
+
+You can find detailed explanations of the properties in [Hive Data Definition Language](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL). For example, the following statement specifies a Kafka spouts and sink:
+
+```
+CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'kafka://localhost:2181/brokers?topic=test' TBLPROPERTIES '{"producer":{"bootstrap.servers":"localhost:9092","acks":"1","key.serializer":"storm.kafka.IntSerializer","value.serializer":"storm.kafka.ByteBufferSerializer"}}'
+```
+
+## Plugging in External Data Sources
+
+Users plug in external data sources through implementing the `ISqlTridentDataSource` interface and registers them using the mechanisms of Java's service loader. The external data source will be chosen based on the scheme of the URI of the tables. Please refer to the implementation of `storm-sql-kafka` for more details.
+
+## Example: Filtering Kafka Stream
+
+Let's say there is a Kafka stream that represents the transactions of orders. Each message in the stream contains the id of the order, the unit price of the product and the quantity of the orders. The goal is to filter orders where the transactions are significant and to insert these orders into another Kafka stream for further analysis.
+
+The user can specify the following SQL statements in the SQL file:
+
+```
+CREATE EXTERNAL TABLE ORDERS (ID INT PRIMARY KEY, UNIT_PRICE INT, QUANTITY INT) LOCATION 'kafka://localhost:2181/brokers?topic=orders' TBLPROPERTIES '{"producer":{"bootstrap.servers":"localhost:9092","acks":"1","key.serializer":"storm.kafka.IntSerializer","value.serializer":"storm.kafka.ByteBufferSerializer"}}'
+CREATE EXTERNAL TABLE LARGE_ORDERS (ID INT PRIMARY KEY, TOTAL INT) LOCATION 'kafka://localhost:2181/brokers?topic=large_orders' TBLPROPERTIES '{"producer":{"bootstrap.servers":"localhost:9092","acks":"1","key.serializer":"storm.kafka.IntSerializer","value.serializer":"storm.kafka.ByteBufferSerializer"}}'
+INSERT INTO LARGE_ORDERS SELECT ID, UNIT_PRICE * QUANTITY AS TOTAL FROM ORDERS WHERE UNIT_PRICE * QUANTITY > 50
+```
+
+The first statement defines the table `ORDER` which represents the input stream. The `LOCATION` clause specifies the ZkHost (`localhost:2181`), the path of the brokers in ZooKeeper (`/brokers`) and the topic (`orders`). The `TBLPROPERTIES` clause specifies the configuration of [KafkaProducer](http://kafka.apache.org/documentation.html#newproducerconfigs).
+Current implementation of `storm-sql-kafka` requires specifying both `LOCATION` and `TBLPROPERTIES` clauses even though the table is read-only or write-only.
+
+Similarly, the second statement specifies the table `LARGE_ORDERS` which represents the output stream. The third statement is a `SELECT` statement which defines the topology: it instructs StormSQL to filter all orders in the external table `ORDERS`, calculates the total price and inserts matching records into the Kafka stream specified by `LARGE_ORDER`.
+
+To run this example, users need to include the data sources (`storm-sql-kafka` in this case) and its dependency in the class path. One approach is to put the required jars into the `extlib` directory:
+
+```
+$ cp curator-client-2.5.0.jar curator-framework-2.5.0.jar zookeeper-3.4.6.jar
+ extlib/
+$ cp scala-library-2.10.4.jar kafka-clients-0.8.2.1.jar kafka_2.10-0.8.2.1.jar metrics-core-2.2.0.jar extlib/
+$ cp json-simple-1.1.1.jar extlib/
+$ cp jackson-annotations-2.6.0.jar extlib/
+$ cp storm-kafka-*.jar storm-sql-kafka-*.jar storm-sql-runtime-*.jar extlib/
+```
+
+The next step is to submit the SQL statements to StormSQL:
+
+```
+$ bin/storm sql order_filtering order_filtering.sql
+```
+
+By now you should be able to see the `order_filtering` topology in the Storm UI.
+
+## Current Limitations
+
+Aggregation, windowing and joining tables are yet to be implemented. Specifying parallelism hints in the topology is not yet supported. All processors have a parallelism hint of 1.
+
+Users also need to provide the dependency of the external data sources in the `extlib` directory. Otherwise the topology will fail to run because of `ClassNotFoundException`.
+
+The current implementation of the Kafka connector in StormSQL assumes both the input and the output are in JSON formats. The connector has not yet recognized the `INPUTFORMAT` and `OUTPUTFORMAT` clauses yet.
[08/10] storm git commit: STORM-1366. Add documentation for StormSQL
into the website.
Posted by wh...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Distributed-RPC.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Distributed-RPC.html b/_site/documentation/Distributed-RPC.html
index c5d137e..b2bbdd2 100644
--- a/_site/documentation/Distributed-RPC.html
+++ b/_site/documentation/Distributed-RPC.html
@@ -97,8 +97,8 @@
<h3 id="high-level-overview">High level overview</h3>
<p>Distributed RPC is coordinated by a "DRPC server" (Storm comes packaged with an implementation of this). The DRPC server coordinates receiving an RPC request, sending the request to the Storm topology, receiving the results from the Storm topology, and sending the results back to the waiting client. From a client's perspective, a distributed RPC call looks just like a regular RPC call. For example, here's how a client would compute the results for the "reach" function with the argument "<a href="http://twitter.com%22:">http://twitter.com":</a></p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">DRPCClient</span> <span class="n">client</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DRPCClient</span><span class="o">(</span><span class="s">"drpc-host"</span><span class="o">,</span> <span class="mi">3772</span><span class="o">);</span>
-<span class="n">String</span> <span class="n">result</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"reach"</span><span class="o">,</span> <span class="s">"http://twitter.com"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">DRPCClient</span> <span class="n">client</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DRPCClient</span><span class="o">(</span><span class="s">"drpc-host"</span><span class="o">,</span> <span class="mi">3772</span><span class="o">);</span>
+<span class="n">String</span> <span class="n">result</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"reach"</span><span class="o">,</span> <span class="s">"http://twitter.com"</span><span class="o">);</span>
</code></pre></div>
<p>The distributed RPC workflow looks like this:</p>
@@ -118,19 +118,19 @@
<p>Let's look at a simple example. Here's the implementation of a DRPC topology that returns its input argument with a "!" appended:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">ExclaimBolt</span> <span class="kd">extends</span> <span class="n">BaseBasicBolt</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">BasicOutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">BasicOutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="n">String</span> <span class="n">input</span> <span class="o">=</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">1</span><span class="o">);</span>
- <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">getValue</span><span class="o">(</span><span class="mi">0</span><span class="o">),</span> <span class="n">input</span> <span class="o">+</span> <span class="s">"!"</span><span class="o">));</span>
+ <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">getValue</span><span class="o">(</span><span class="mi">0</span><span class="o">),</span> <span class="n">input</span> <span class="o">+</span> <span class="s">"!"</span><span class="o">));</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"result"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"result"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
-<span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span>
- <span class="n">LinearDRPCTopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">LinearDRPCTopologyBuilder</span><span class="o">(</span><span class="s">"exclamation"</span><span class="o">);</span>
- <span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="nf">ExclaimBolt</span><span class="o">(),</span> <span class="mi">3</span><span class="o">);</span>
+<span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="p">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span>
+ <span class="n">LinearDRPCTopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LinearDRPCTopologyBuilder</span><span class="o">(</span><span class="s">"exclamation"</span><span class="o">);</span>
+ <span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="n">ExclaimBolt</span><span class="o">(),</span> <span class="mi">3</span><span class="o">);</span>
<span class="c1">// ...</span>
<span class="o">}</span>
</code></pre></div>
@@ -141,12 +141,12 @@
<h3 id="local-mode-drpc">Local mode DRPC</h3>
<p>DRPC can be run in local mode. Here's how to run the above example in local mode:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">LocalDRPC</span> <span class="n">drpc</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">LocalDRPC</span><span class="o">();</span>
-<span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">LocalCluster</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">LocalDRPC</span> <span class="n">drpc</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LocalDRPC</span><span class="o">();</span>
+<span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LocalCluster</span><span class="o">();</span>
-<span class="n">cluster</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"drpc-demo"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createLocalTopology</span><span class="o">(</span><span class="n">drpc</span><span class="o">));</span>
+<span class="n">cluster</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"drpc-demo"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createLocalTopology</span><span class="o">(</span><span class="n">drpc</span><span class="o">));</span>
-<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">"Results for 'hello':"</span> <span class="o">+</span> <span class="n">drpc</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"exclamation"</span><span class="o">,</span> <span class="s">"hello"</span><span class="o">));</span>
+<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">"Results for 'hello':"</span> <span class="o">+</span> <span class="n">drpc</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"exclamation"</span><span class="o">,</span> <span class="s">"hello"</span><span class="o">));</span>
<span class="n">cluster</span><span class="o">.</span><span class="na">shutdown</span><span class="o">();</span>
<span class="n">drpc</span><span class="o">.</span><span class="na">shutdown</span><span class="o">();</span>
@@ -166,15 +166,15 @@
</ol>
<p>Launching a DRPC server can be done with the <code>storm</code> script and is just like launching Nimbus or the UI:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">bin/storm drpc
+<div class="highlight"><pre><code class="language-" data-lang="">bin/storm drpc
</code></pre></div>
<p>Next, you need to configure your Storm cluster to know the locations of the DRPC server(s). This is how <code>DRPCSpout</code> knows from where to read function invocations. This can be done through the <code>storm.yaml</code> file or the topology configurations. Configuring this through the <code>storm.yaml</code> looks something like this:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">drpc.servers</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="s">"drpc1.foo.com"</span>
- <span class="p-Indicator">-</span> <span class="s">"drpc2.foo.com"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">drpc.servers</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">drpc1.foo.com"</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">drpc2.foo.com"</span>
</code></pre></div>
<p>Finally, you launch DRPC topologies using <code>StormSubmitter</code> just like you launch any other topology. To run the above example in remote mode, you do something like this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">StormSubmitter</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"exclamation-drpc"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createRemoteTopology</span><span class="o">());</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">StormSubmitter</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"exclamation-drpc"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createRemoteTopology</span><span class="o">());</span>
</code></pre></div>
<p><code>createRemoteTopology</code> is used to create topologies suitable for Storm clusters.</p>
@@ -194,14 +194,14 @@
<p>A single reach computation can involve thousands of database calls and tens of millions of follower records during the computation. It's a really, really intense computation. As you're about to see, implementing this function on top of Storm is dead simple. On a single machine, reach can take minutes to compute; on a Storm cluster, you can compute reach for even the hardest URLs in a couple seconds.</p>
<p>A sample reach topology is defined in storm-starter <a href="https://github.com/apache/storm/blob/master/examples/storm-starter/src/jvm/storm/starter/ReachTopology.java">here</a>. Here's how you define the reach topology:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">LinearDRPCTopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">LinearDRPCTopologyBuilder</span><span class="o">(</span><span class="s">"reach"</span><span class="o">);</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="nf">GetTweeters</span><span class="o">(),</span> <span class="mi">3</span><span class="o">);</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="nf">GetFollowers</span><span class="o">(),</span> <span class="mi">12</span><span class="o">)</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">LinearDRPCTopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LinearDRPCTopologyBuilder</span><span class="o">(</span><span class="s">"reach"</span><span class="o">);</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="n">GetTweeters</span><span class="o">(),</span> <span class="mi">3</span><span class="o">);</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="n">GetFollowers</span><span class="o">(),</span> <span class="mi">12</span><span class="o">)</span>
<span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">();</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="nf">PartialUniquer</span><span class="o">(),</span> <span class="mi">6</span><span class="o">)</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"follower"</span><span class="o">));</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="nf">CountAggregator</span><span class="o">(),</span> <span class="mi">2</span><span class="o">)</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">));</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="n">PartialUniquer</span><span class="o">(),</span> <span class="mi">6</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"follower"</span><span class="o">));</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">addBolt</span><span class="o">(</span><span class="k">new</span> <span class="n">CountAggregator</span><span class="o">(),</span> <span class="mi">2</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">));</span>
</code></pre></div>
<p>The topology executes as four steps:</p>
@@ -219,24 +219,24 @@
<span class="n">Set</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">_followers</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HashSet</span><span class="o"><</span><span class="n">String</span><span class="o">>();</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">BatchOutputCollector</span> <span class="n">collector</span><span class="o">,</span> <span class="n">Object</span> <span class="n">id</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">BatchOutputCollector</span> <span class="n">collector</span><span class="o">,</span> <span class="n">Object</span> <span class="n">id</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_collector</span> <span class="o">=</span> <span class="n">collector</span><span class="o">;</span>
<span class="n">_id</span> <span class="o">=</span> <span class="n">id</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_followers</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">1</span><span class="o">));</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">finishBatch</span><span class="o">()</span> <span class="o">{</span>
- <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">_id</span><span class="o">,</span> <span class="n">_followers</span><span class="o">.</span><span class="na">size</span><span class="o">()));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">finishBatch</span><span class="o">()</span> <span class="o">{</span>
+ <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">_id</span><span class="o">,</span> <span class="n">_followers</span><span class="o">.</span><span class="na">size</span><span class="o">()));</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"partial-count"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"partial-count"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/FAQ.html
----------------------------------------------------------------------
diff --git a/_site/documentation/FAQ.html b/_site/documentation/FAQ.html
index 06f7e52..e9781a0 100644
--- a/_site/documentation/FAQ.html
+++ b/_site/documentation/FAQ.html
@@ -92,7 +92,7 @@
<h2 id="best-practices">Best Practices</h2>
-<h3 id="what-rules-of-thumb-can-you-give-me-for-configuring-storm+trident?">What rules of thumb can you give me for configuring Storm+Trident?</h3>
+<h3 id="what-rules-of-thumb-can-you-give-me-for-configuring-storm-trident">What rules of thumb can you give me for configuring Storm+Trident?</h3>
<ul>
<li>number of workers a multiple of number of machines; parallelism a multiple of number of workers; number of kafka partitions a multiple of number of spout parallelism</li>
@@ -105,7 +105,7 @@
<li>Start with a max spout pending that is for sure too small -- one for trident, or the number of executors for storm -- and increase it until you stop seeing changes in the flow. You'll probably end up with something near <code>2*(throughput in recs/sec)*(end-to-end latency)</code> (2x the Little's law capacity).</li>
</ul>
-<h3 id="what-are-some-of-the-best-ways-to-get-a-worker-to-mysteriously-and-bafflingly-die?">What are some of the best ways to get a worker to mysteriously and bafflingly die?</h3>
+<h3 id="what-are-some-of-the-best-ways-to-get-a-worker-to-mysteriously-and-bafflingly-die">What are some of the best ways to get a worker to mysteriously and bafflingly die?</h3>
<ul>
<li>Do you have write access to the log directory</li>
@@ -116,7 +116,7 @@
<li>Have you opened firewall/securitygroup permissions <em>bidirectionally</em> among a) all the workers, b) the storm master, c) zookeeper? Also, from the workers to any kafka/kestrel/database/etc that your topology accesses? Use netcat to poke the appropriate ports and be sure. </li>
</ul>
-<h3 id="halp!-i-cannot-see:">Halp! I cannot see:</h3>
+<h3 id="halp-i-cannot-see">Halp! I cannot see:</h3>
<ul>
<li><strong>my logs</strong> Logs by default go to $STORM_HOME/logs. Check that you have write permissions to that directory. They are configured in
@@ -130,7 +130,7 @@
<li><strong>final Java system properties</strong> Add <code>Properties props = System.getProperties(); props.list(System.out);</code> near where you build your topology.</li>
</ul>
-<h3 id="how-many-workers-should-i-use?">How many Workers should I use?</h3>
+<h3 id="how-many-workers-should-i-use">How many Workers should I use?</h3>
<p>The total number of workers is set by the supervisors -- there's some number of JVM slots each supervisor will superintend. The thing you set on the topology is how many worker slots it will try to claim.</p>
@@ -146,14 +146,14 @@
<h2 id="topology">Topology</h2>
-<h3 id="can-a-trident-topology-have-multiple-streams?">Can a Trident topology have Multiple Streams?</h3>
+<h3 id="can-a-trident-topology-have-multiple-streams">Can a Trident topology have Multiple Streams?</h3>
<blockquote>
<p>Can a Trident Topology work like a workflow with conditional paths (if-else)? e.g. A Spout (S1) connects to a bolt (B0) which based on certain values in the incoming tuple routes them to either bolt (B1) or bolt (B2) but not both.</p>
</blockquote>
<p>A Trident "each" operator returns a Stream object, which you can store in a variable. You can then run multiple eaches on the same Stream to split it, e.g.: </p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"> Stream s = topology.each(...).groupBy(...).aggregate(...)
+<div class="highlight"><pre><code class="language-" data-lang=""> Stream s = topology.each(...).groupBy(...).aggregate(...)
Stream branch1 = s.each(..., FilterA)
Stream branch2 = s.each(..., FilterB)
</code></pre></div>
@@ -163,23 +163,23 @@
<h2 id="spouts">Spouts</h2>
-<h3 id="what-is-a-coordinator,-and-why-are-there-several?">What is a coordinator, and why are there several?</h3>
+<h3 id="what-is-a-coordinator-and-why-are-there-several">What is a coordinator, and why are there several?</h3>
<p>A trident-spout is actually run within a storm <em>bolt</em>. The storm-spout of a trident topology is the MasterBatchCoordinator -- it coordinates trident batches and is the same no matter what spouts you use. A batch is born when the MBC dispenses a seed tuple to each of the spout-coordinators. The spout-coordinator bolts know how your particular spouts should cooperate -- so in the kafka case, it's what helps figure out what partition and offset range each spout should pull from.</p>
-<h3 id="what-can-i-store-into-the-spout's-metadata-record?">What can I store into the spout's metadata record?</h3>
+<h3 id="what-can-i-store-into-the-spout-39-s-metadata-record">What can I store into the spout's metadata record?</h3>
<p>You should only store static data, and as little of it as possible, into the metadata record (note: maybe you <em>can</em> store more interesting things; you shouldn't, though)</p>
-<h3 id="how-often-is-the-'emitpartitionbatchnew'-function-called?">How often is the 'emitPartitionBatchNew' function called?</h3>
+<h3 id="how-often-is-the-39-emitpartitionbatchnew-39-function-called">How often is the 'emitPartitionBatchNew' function called?</h3>
<p>Since the MBC is the actual spout, all the tuples in a batch are just members of its tupletree. That means storm's "max spout pending" config effectively defines the number of concurrent batches trident runs. The MBC emits a new batch if it has fewer than max-spending tuples pending and if at least one <a href="https://github.com/apache/storm/blob/master/conf/defaults.yaml#L115">trident batch interval</a>'s worth of seconds has passed since the last batch.</p>
-<h3 id="if-nothing-was-emitted-does-trident-slow-down-the-calls?">If nothing was emitted does Trident slow down the calls?</h3>
+<h3 id="if-nothing-was-emitted-does-trident-slow-down-the-calls">If nothing was emitted does Trident slow down the calls?</h3>
<p>Yes, there's a pluggable "spout wait strategy"; the default is to sleep for a <a href="https://github.com/apache/storm/blob/master/conf/defaults.yaml#L110">configurable amount of time</a></p>
-<h3 id="ok,-then-what-is-the-trident-batch-interval-for?">OK, then what is the trident batch interval for?</h3>
+<h3 id="ok-then-what-is-the-trident-batch-interval-for">OK, then what is the trident batch interval for?</h3>
<p>You know how computers of the 486 era had a <a href="http://en.wikipedia.org/wiki/Turbo_button">turbo button</a> on them? It's like that. </p>
@@ -189,11 +189,11 @@
<p>Note that this is a cap, not an additional delay -- with a period of 300ms, if your batch takes 258ms Trident will only delay an additional 42ms.</p>
-<h3 id="how-do-you-set-the-batch-size?">How do you set the batch size?</h3>
+<h3 id="how-do-you-set-the-batch-size">How do you set the batch size?</h3>
<p>Trident doesn't place its own limits on the batch count. In the case of the Kafka spout, the max fetch bytes size divided by the average record size defines an effective records per subbatch partition.</p>
-<h3 id="how-do-i-resize-a-batch?">How do I resize a batch?</h3>
+<h3 id="how-do-i-resize-a-batch">How do I resize a batch?</h3>
<p>The trident batch is a somewhat overloaded facility. Together with the number of partitions, the batch size is constrained by or serves to define</p>
@@ -208,13 +208,13 @@
<h2 id="time-series">Time Series</h2>
-<h3 id="how-do-i-aggregate-events-by-time?">How do I aggregate events by time?</h3>
+<h3 id="how-do-i-aggregate-events-by-time">How do I aggregate events by time?</h3>
<p>If have records with an immutable timestamp, and you would like to count, average or otherwise aggregate them into discrete time buckets, Trident is an excellent and scalable solution.</p>
<p>Write an <code>Each</code> function that turns the timestamp into a time bucket: if the bucket size was "by hour", then the timestamp <code>2013-08-08 12:34:56</code> would be mapped to the <code>2013-08-08 12:00:00</code> time bucket, and so would everything else in the twelve o'clock hour. Then group on that timebucket and use a grouped persistentAggregate. The persistentAggregate uses a local cacheMap backed by a data store. Groups with many records require very few reads from the data store, and use efficient bulk reads and writes; as long as your data feed is relatively prompt Trident will make very efficient use of memory and network. Even if a server drops off line for a day, then delivers that full day's worth of data in a rush, the old results will be calmly retrieved and updated -- and without interfering with calculating the current results.</p>
-<h3 id="how-can-i-know-that-all-records-for-a-time-bucket-have-been-received?">How can I know that all records for a time bucket have been received?</h3>
+<h3 id="how-can-i-know-that-all-records-for-a-time-bucket-have-been-received">How can I know that all records for a time bucket have been received?</h3>
<p>You cannot know that all events are collected -- this is an epistemological challenge, not a distributed systems challenge. You can:</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Fault-tolerance.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Fault-tolerance.html b/_site/documentation/Fault-tolerance.html
index f1ca246..9ce6412 100644
--- a/_site/documentation/Fault-tolerance.html
+++ b/_site/documentation/Fault-tolerance.html
@@ -92,27 +92,27 @@
<p>This page explains the design details of Storm that make it a fault-tolerant system.</p>
-<h2 id="what-happens-when-a-worker-dies?">What happens when a worker dies?</h2>
+<h2 id="what-happens-when-a-worker-dies">What happens when a worker dies?</h2>
<p>When a worker dies, the supervisor will restart it. If it continuously fails on startup and is unable to heartbeat to Nimbus, Nimbus will reassign the worker to another machine.</p>
-<h2 id="what-happens-when-a-node-dies?">What happens when a node dies?</h2>
+<h2 id="what-happens-when-a-node-dies">What happens when a node dies?</h2>
<p>The tasks assigned to that machine will time-out and Nimbus will reassign those tasks to other machines.</p>
-<h2 id="what-happens-when-nimbus-or-supervisor-daemons-die?">What happens when Nimbus or Supervisor daemons die?</h2>
+<h2 id="what-happens-when-nimbus-or-supervisor-daemons-die">What happens when Nimbus or Supervisor daemons die?</h2>
<p>The Nimbus and Supervisor daemons are designed to be fail-fast (process self-destructs whenever any unexpected situation is encountered) and stateless (all state is kept in Zookeeper or on disk). As described in <a href="Setting-up-a-Storm-cluster.html">Setting up a Storm cluster</a>, the Nimbus and Supervisor daemons must be run under supervision using a tool like daemontools or monit. So if the Nimbus or Supervisor daemons die, they restart like nothing happened.</p>
<p>Most notably, no worker processes are affected by the death of Nimbus or the Supervisors. This is in contrast to Hadoop, where if the JobTracker dies, all the running jobs are lost. </p>
-<h2 id="is-nimbus-a-single-point-of-failure?">Is Nimbus a single point of failure?</h2>
+<h2 id="is-nimbus-a-single-point-of-failure">Is Nimbus a single point of failure?</h2>
<p>If you lose the Nimbus node, the workers will still continue to function. Additionally, supervisors will continue to restart workers if they die. However, without Nimbus, workers won't be reassigned to other machines when necessary (like if you lose a worker machine). </p>
<p>So the answer is that Nimbus is "sort of" a SPOF. In practice, it's not a big deal since nothing catastrophic happens when the Nimbus daemon dies. There are plans to make Nimbus highly available in the future.</p>
-<h2 id="how-does-storm-guarantee-data-processing?">How does Storm guarantee data processing?</h2>
+<h2 id="how-does-storm-guarantee-data-processing">How does Storm guarantee data processing?</h2>
<p>Storm provides mechanisms to guarantee data processing even if nodes die or messages are lost. See <a href="Guaranteeing-message-processing.html">Guaranteeing message processing</a> for the details.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Guaranteeing-message-processing.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Guaranteeing-message-processing.html b/_site/documentation/Guaranteeing-message-processing.html
index fb707a1..1ec0ce1 100644
--- a/_site/documentation/Guaranteeing-message-processing.html
+++ b/_site/documentation/Guaranteeing-message-processing.html
@@ -92,18 +92,18 @@
<p>Storm guarantees that each message coming off a spout will be fully processed. This page describes how Storm accomplishes this guarantee and what you have to do as a user to benefit from Storm's reliability capabilities.</p>
-<h3 id="what-does-it-mean-for-a-message-to-be-"fully-processed"?">What does it mean for a message to be "fully processed"?</h3>
+<h3 id="what-does-it-mean-for-a-message-to-be-quot-fully-processed-quot">What does it mean for a message to be "fully processed"?</h3>
<p>A tuple coming off a spout can trigger thousands of tuples to be created based on it. Consider, for example, the streaming word count topology:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TopologyBuilder</span><span class="o">();</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">KestrelSpout</span><span class="o">(</span><span class="s">"kestrel.backtype.com"</span><span class="o">,</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TopologyBuilder</span><span class="o">();</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">,</span> <span class="k">new</span> <span class="n">KestrelSpout</span><span class="o">(</span><span class="s">"kestrel.backtype.com"</span><span class="o">,</span>
<span class="mi">22133</span><span class="o">,</span>
- <span class="s">"sentence_queue"</span><span class="o">,</span>
- <span class="k">new</span> <span class="nf">StringScheme</span><span class="o">()));</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">SplitSentence</span><span class="o">(),</span> <span class="mi">10</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">);</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"count"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">WordCount</span><span class="o">(),</span> <span class="mi">20</span><span class="o">)</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
+ <span class="s">"sentence_queue"</span><span class="o">,</span>
+ <span class="k">new</span> <span class="n">StringScheme</span><span class="o">()));</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="n">SplitSentence</span><span class="o">(),</span> <span class="mi">10</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">);</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"count"</span><span class="o">,</span> <span class="k">new</span> <span class="n">WordCount</span><span class="o">(),</span> <span class="mi">20</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
</code></pre></div>
<p>This topology reads sentences off of a Kestrel queue, splits the sentences into its constituent words, and then emits for each word the number of times it has seen that word before. A tuple coming off the spout triggers many tuples being created based on it: a tuple for each word in the sentence and a tuple for the updated count for each word. The tree of messages looks something like this:</p>
@@ -111,25 +111,25 @@
<p>Storm considers a tuple coming off a spout "fully processed" when the tuple tree has been exhausted and every message in the tree has been processed. A tuple is considered failed when its tree of messages fails to be fully processed within a specified timeout. This timeout can be configured on a topology-specific basis using the <a href="/javadoc/apidocs/backtype/storm/Config.html#TOPOLOGY_MESSAGE_TIMEOUT_SECS">Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS</a> configuration and defaults to 30 seconds.</p>
-<h3 id="what-happens-if-a-message-is-fully-processed-or-fails-to-be-fully-processed?">What happens if a message is fully processed or fails to be fully processed?</h3>
+<h3 id="what-happens-if-a-message-is-fully-processed-or-fails-to-be-fully-processed">What happens if a message is fully processed or fails to be fully processed?</h3>
<p>To understand this question, let's take a look at the lifecycle of a tuple coming off of a spout. For reference, here is the interface that spouts implement (see the <a href="/javadoc/apidocs/backtype/storm/spout/ISpout.html">Javadoc</a> for more information):</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">ISpout</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="kt">void</span> <span class="nf">open</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">SpoutOutputCollector</span> <span class="n">collector</span><span class="o">);</span>
- <span class="kt">void</span> <span class="nf">close</span><span class="o">();</span>
- <span class="kt">void</span> <span class="nf">nextTuple</span><span class="o">();</span>
- <span class="kt">void</span> <span class="nf">ack</span><span class="o">(</span><span class="n">Object</span> <span class="n">msgId</span><span class="o">);</span>
- <span class="kt">void</span> <span class="nf">fail</span><span class="o">(</span><span class="n">Object</span> <span class="n">msgId</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">open</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">SpoutOutputCollector</span> <span class="n">collector</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">close</span><span class="o">();</span>
+ <span class="kt">void</span> <span class="n">nextTuple</span><span class="o">();</span>
+ <span class="kt">void</span> <span class="n">ack</span><span class="o">(</span><span class="n">Object</span> <span class="n">msgId</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">fail</span><span class="o">(</span><span class="n">Object</span> <span class="n">msgId</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>First, Storm requests a tuple from the <code>Spout</code> by calling the <code>nextTuple</code> method on the <code>Spout</code>. The <code>Spout</code> uses the <code>SpoutOutputCollector</code> provided in the <code>open</code> method to emit a tuple to one of its output streams. When emitting a tuple, the <code>Spout</code> provides a "message id" that will be used to identify the tuple later. For example, the <code>KestrelSpout</code> reads a message off of the kestrel queue and emits as the "message id" the id provided by Kestrel for the message. Emitting a message to the <code>SpoutOutputCollector</code> looks like this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="s">"field1"</span><span class="o">,</span> <span class="s">"field2"</span><span class="o">,</span> <span class="mi">3</span><span class="o">)</span> <span class="o">,</span> <span class="n">msgId</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="s">"field1"</span><span class="o">,</span> <span class="s">"field2"</span><span class="o">,</span> <span class="mi">3</span><span class="o">)</span> <span class="o">,</span> <span class="n">msgId</span><span class="o">);</span>
</code></pre></div>
<p>Next, the tuple gets sent to consuming bolts and Storm takes care of tracking the tree of messages that is created. If Storm detects that a tuple is fully processed, Storm will call the <code>ack</code> method on the originating <code>Spout</code> task with the message id that the <code>Spout</code> provided to Storm. Likewise, if the tuple times-out Storm will call the <code>fail</code> method on the <code>Spout</code>. Note that a tuple will be acked or failed by the exact same <code>Spout</code> task that created it. So if a <code>Spout</code> is executing as many tasks across the cluster, a tuple won't be acked or failed by a different task than the one that created it.</p>
<p>Let's use <code>KestrelSpout</code> again to see what a <code>Spout</code> needs to do to guarantee message processing. When <code>KestrelSpout</code> takes a message off the Kestrel queue, it "opens" the message. This means the message is not actually taken off the queue yet, but instead placed in a "pending" state waiting for acknowledgement that the message is completed. While in the pending state, a message will not be sent to other consumers of the queue. Additionally, if a client disconnects all pending messages for that client are put back on the queue. When a message is opened, Kestrel provides the client with the data for the message as well as a unique id for the message. The <code>KestrelSpout</code> uses that exact id as the "message id" for the tuple when emitting the tuple to the <code>SpoutOutputCollector</code>. Sometime later on, when <code>ack</code> or <code>fail</code> are called on the <code>KestrelSpout</code>, the <code>Kes
trelSpout</code> sends an ack or fail message to Kestrel with the message id to take the message off the queue or have it put back on.</p>
-<h3 id="what-is-storm's-reliability-api?">What is Storm's reliability API?</h3>
+<h3 id="what-is-storm-39-s-reliability-api">What is Storm's reliability API?</h3>
<p>There's two things you have to do as a user to benefit from Storm's reliability capabilities. First, you need to tell Storm whenever you're creating a new link in the tree of tuples. Second, you need to tell Storm when you have finished processing an individual tuple. By doing both these things, Storm can detect when the tree of tuples is fully processed and can ack or fail the spout tuple appropriately. Storm's API provides a concise way of doing both of these tasks. </p>
@@ -137,25 +137,25 @@
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">SplitSentence</span> <span class="kd">extends</span> <span class="n">BaseRichBolt</span> <span class="o">{</span>
<span class="n">OutputCollector</span> <span class="n">_collector</span><span class="o">;</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">OutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">OutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_collector</span> <span class="o">=</span> <span class="n">collector</span><span class="o">;</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="n">String</span> <span class="n">sentence</span> <span class="o">=</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span>
- <span class="k">for</span><span class="o">(</span><span class="n">String</span> <span class="nl">word:</span> <span class="n">sentence</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">))</span> <span class="o">{</span>
- <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">tuple</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
+ <span class="k">for</span><span class="o">(</span><span class="n">String</span> <span class="nl">word:</span> <span class="n">sentence</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">))</span> <span class="o">{</span>
+ <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">tuple</span><span class="o">,</span> <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
<span class="o">}</span>
<span class="n">_collector</span><span class="o">.</span><span class="na">ack</span><span class="o">(</span><span class="n">tuple</span><span class="o">);</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
<p>Each word tuple is <em>anchored</em> by specifying the input tuple as the first argument to <code>emit</code>. Since the word tuple is anchored, the spout tuple at the root of the tree will be replayed later on if the word tuple failed to be processed downstream. In contrast, let's look at what happens if the word tuple is emitted like this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
</code></pre></div>
<p>Emitting the word tuple this way causes it to be <em>unanchored</em>. If the tuple fails be processed downstream, the root tuple will not be replayed. Depending on the fault-tolerance guarantees you need in your topology, sometimes it's appropriate to emit an unanchored tuple.</p>
@@ -163,7 +163,7 @@
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">List</span><span class="o"><</span><span class="n">Tuple</span><span class="o">></span> <span class="n">anchors</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o"><</span><span class="n">Tuple</span><span class="o">>();</span>
<span class="n">anchors</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="n">tuple1</span><span class="o">);</span>
<span class="n">anchors</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="n">tuple2</span><span class="o">);</span>
-<span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">anchors</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">2</span><span class="o">,</span> <span class="mi">3</span><span class="o">));</span>
+<span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">anchors</span><span class="o">,</span> <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">2</span><span class="o">,</span> <span class="mi">3</span><span class="o">));</span>
</code></pre></div>
<p>Multi-anchoring adds the output tuple into multiple tuple trees. Note that it's also possible for multi-anchoring to break the tree structure and create tuple DAGs, like so:</p>
@@ -179,15 +179,15 @@
<p>A lot of bolts follow a common pattern of reading an input tuple, emitting tuples based on it, and then acking the tuple at the end of the <code>execute</code> method. These bolts fall into the categories of filters and simple functions. Storm has an interface called <code>BasicBolt</code> that encapsulates this pattern for you. The <code>SplitSentence</code> example can be written as a <code>BasicBolt</code> like follows:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">SplitSentence</span> <span class="kd">extends</span> <span class="n">BaseBasicBolt</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">BasicOutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">BasicOutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="n">String</span> <span class="n">sentence</span> <span class="o">=</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span>
- <span class="k">for</span><span class="o">(</span><span class="n">String</span> <span class="nl">word:</span> <span class="n">sentence</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">))</span> <span class="o">{</span>
- <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
+ <span class="k">for</span><span class="o">(</span><span class="n">String</span> <span class="nl">word:</span> <span class="n">sentence</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">))</span> <span class="o">{</span>
+ <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
@@ -195,11 +195,11 @@
<p>In contrast, bolts that do aggregations or joins may delay acking a tuple until after it has computed a result based on a bunch of tuples. Aggregations and joins will commonly multi-anchor their output tuples as well. These things fall outside the simpler pattern of <code>IBasicBolt</code>.</p>
-<h3 id="how-do-i-make-my-applications-work-correctly-given-that-tuples-can-be-replayed?">How do I make my applications work correctly given that tuples can be replayed?</h3>
+<h3 id="how-do-i-make-my-applications-work-correctly-given-that-tuples-can-be-replayed">How do I make my applications work correctly given that tuples can be replayed?</h3>
<p>As always in software design, the answer is "it depends." Storm 0.7.0 introduced the "transactional topologies" feature, which enables you to get fully fault-tolerant exactly-once messaging semantics for most computations. Read more about transactional topologies <a href="Transactional-topologies.html">here</a>. </p>
-<h3 id="how-does-storm-implement-reliability-in-an-efficient-way?">How does Storm implement reliability in an efficient way?</h3>
+<h3 id="how-does-storm-implement-reliability-in-an-efficient-way">How does Storm implement reliability in an efficient way?</h3>
<p>A Storm topology has a set of special "acker" tasks that track the DAG of tuples for every spout tuple. When an acker sees that a DAG is complete, it sends a message to the spout task that created the spout tuple to ack the message. You can set the number of acker executors for a topology in the topology configuration using <a href="/javadoc/apidocs/backtype/storm/Config.html#TOPOLOGY_ACKER_EXECUTORS">Config.TOPOLOGY_ACKER_EXECUTORS</a>. Storm defaults TOPOLOGY_ACKER_EXECUTORS to be equal to the number of workers configured in the topology -- you will need to increase this number for topologies processing large amounts of messages.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Home.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Home.html b/_site/documentation/Home.html
index 15d59f1..c3813f4 100644
--- a/_site/documentation/Home.html
+++ b/_site/documentation/Home.html
@@ -130,13 +130,13 @@
<p>You can view the archives of the mailing list <a href="http://mail-archives.apache.org/mod_mbox/storm-dev/">here</a>.</p>
-<h4 id="which-list-should-i-send/subscribe-to?">Which list should I send/subscribe to?</h4>
+<h4 id="which-list-should-i-send-subscribe-to">Which list should I send/subscribe to?</h4>
<p>If you are using a pre-built binary distribution of Storm, then chances are you should send questions, comments, storm-related announcements, etc. to <a href="user@storm.apache.org">user@storm.apache.org</a>. </p>
<p>If you are building storm from source, developing new features, or otherwise hacking storm source code, then <a href="dev@storm.apache.org">dev@storm.apache.org</a> is more appropriate. </p>
-<h4 id="what-will-happen-with-storm-user@googlegroups.com?">What will happen with <a href="mailto:storm-user@googlegroups.com">storm-user@googlegroups.com</a>?</h4>
+<h4 id="what-will-happen-with-storm-user-googlegroups-com">What will happen with <a href="mailto:storm-user@googlegroups.com">storm-user@googlegroups.com</a>?</h4>
<p>All existing messages will remain archived there, and can be accessed/searched <a href="https://groups.google.com/forum/#!forum/storm-user">here</a>.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Installing-native-dependencies.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Installing-native-dependencies.html b/_site/documentation/Installing-native-dependencies.html
index 37a0589..4378012 100644
--- a/_site/documentation/Installing-native-dependencies.html
+++ b/_site/documentation/Installing-native-dependencies.html
@@ -95,7 +95,7 @@
<p>Installing ZeroMQ and JZMQ is usually straightforward. Sometimes, however, people run into issues with autoconf and get strange errors. If you run into any issues, please email the <a href="http://groups.google.com/group/storm-user">Storm mailing list</a> or come get help in the #storm-user room on freenode. </p>
<p>Storm has been tested with ZeroMQ 2.1.7, and this is the recommended ZeroMQ release that you install. You can download a ZeroMQ release <a href="http://download.zeromq.org/">here</a>. Installing ZeroMQ should look something like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">wget http://download.zeromq.org/zeromq-2.1.7.tar.gz
+<div class="highlight"><pre><code class="language-" data-lang="">wget http://download.zeromq.org/zeromq-2.1.7.tar.gz
tar -xzf zeromq-2.1.7.tar.gz
cd zeromq-2.1.7
./configure
@@ -103,7 +103,7 @@ make
sudo make install
</code></pre></div>
<p>JZMQ is the Java bindings for ZeroMQ. JZMQ doesn't have any releases (we're working with them on that), so there is risk of a regression if you always install from the master branch. To prevent a regression from happening, you should instead install from <a href="http://github.com/nathanmarz/jzmq">this fork</a> which is tested to work with Storm. Installing JZMQ should look something like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">#install jzmq
+<div class="highlight"><pre><code class="language-" data-lang="">#install jzmq
git clone https://github.com/nathanmarz/jzmq.git
cd jzmq
./autogen.sh
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Kestrel-and-Storm.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Kestrel-and-Storm.html b/_site/documentation/Kestrel-and-Storm.html
index 16b7717..424783e 100644
--- a/_site/documentation/Kestrel-and-Storm.html
+++ b/_site/documentation/Kestrel-and-Storm.html
@@ -121,15 +121,15 @@
<h2 id="add-items-to-kestrel">Add items to Kestrel</h2>
<p>At first, we need to have a program that can add items to a Kestrel queue. The following method takes benefit of the KestrelClient implementation in <a href="https://github.com/nathanmarz/storm-kestrel">storm-kestrel</a>. It adds sentences into a Kestrel queue randomly chosen out of an array that holds five possible sentences.</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"> private static void queueSentenceItems(KestrelClient kestrelClient, String queueName)
+<div class="highlight"><pre><code class="language-" data-lang=""> private static void queueSentenceItems(KestrelClient kestrelClient, String queueName)
throws ParseError, IOException {
String[] sentences = new String[] {
- "the cow jumped over the moon",
- "an apple a day keeps the doctor away",
- "four score and seven years ago",
- "snow white and the seven dwarfs",
- "i am at two with nature"};
+ "the cow jumped over the moon",
+ "an apple a day keeps the doctor away",
+ "four score and seven years ago",
+ "snow white and the seven dwarfs",
+ "i am at two with nature"};
Random _rand = new Random();
@@ -137,11 +137,11 @@
String sentence = sentences[_rand.nextInt(sentences.length)];
- String val = "ID " + i + " " + sentence;
+ String val = "ID " + i + " " + sentence;
boolean queueSucess = kestrelClient.queue(queueName, val);
- System.out.println("queueSucess=" +queueSucess+ " [" + val +"]");
+ System.out.println("queueSucess=" +queueSucess+ " [" + val +"]");
}
}
</code></pre></div>
@@ -152,10 +152,10 @@
private static void dequeueItems(KestrelClient kestrelClient, String queueName) throws IOException, ParseError
{
for(int i=1; i<=12; i++){</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"> Item item = kestrelClient.dequeue(queueName);
+<div class="highlight"><pre><code class="language-" data-lang=""> Item item = kestrelClient.dequeue(queueName);
if(item==null){
- System.out.println("The queue (" + queueName + ") contains no items.");
+ System.out.println("The queue (" + queueName + ") contains no items.");
}
else
{
@@ -163,11 +163,12 @@
String receivedVal = new String(data);
- System.out.println("receivedItem=" + receivedVal);
+ System.out.println("receivedItem=" + receivedVal);
}
}
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">This method dequeues items from a queue and then removes them.
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">private static void dequeueAndRemoveItems(KestrelClient kestrelClient, String queueName)
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">
+This method dequeues items from a queue and then removes them.
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">private static void dequeueAndRemoveItems(KestrelClient kestrelClient, String queueName)
throws IOException, ParseError
{
for(int i=1; i<=12; i++){
@@ -176,7 +177,7 @@ throws IOException, ParseError
if(item==null){
- System.out.println("The queue (" + queueName + ") contains no items.");
+ System.out.println("The queue (" + queueName + ") contains no items.");
}
else
{
@@ -189,16 +190,18 @@ throws IOException, ParseError
kestrelClient.ack(queueName, itemID);
- System.out.println("receivedItem=" + receivedVal);
+ System.out.println("receivedItem=" + receivedVal);
}
}
}
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">## Add Items continuously to Kestrel
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">
+## Add Items continuously to Kestrel
This is our final program to run in order to add continuously sentence items to a queue called **sentence_queue** of a locally running Kestrel server.
-In order to stop it type a closing bracket char ']' in console and hit 'Enter'.
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">import java.io.IOException;
+In order to stop it type a closing bracket char ']' in console and hit 'Enter'.
+
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">import java.io.IOException;
import java.io.InputStream;
import java.util.Random;
@@ -215,7 +218,7 @@ public class AddSentenceItemsToKestrel {
InputStream is = System.in;
- char closing_bracket = ']';
+ char closing_bracket = ']';
int val = closing_bracket;
@@ -224,11 +227,11 @@ public class AddSentenceItemsToKestrel {
try {
KestrelClient kestrelClient = null;
- String queueName = "sentence_queue";
+ String queueName = "sentence_queue";
while(aux){
- kestrelClient = new KestrelClient("localhost",22133);
+ kestrelClient = new KestrelClient("localhost",22133);
queueSentenceItems(kestrelClient, queueName);
@@ -253,20 +256,22 @@ public class AddSentenceItemsToKestrel {
e.printStackTrace();
}
- System.out.println("end");
+ System.out.println("end");
}
}
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">## Using KestrelSpout
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">## Using KestrelSpout
This topology reads sentences off of a Kestrel queue using KestrelSpout, splits the sentences into its constituent words (Bolt: SplitSentence), and then emits for each word the number of times it has seen that word before (Bolt: WordCount). How data is processed is described in detail in [Guaranteeing message processing](Guaranteeing-message-processing.html).
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">TopologyBuilder builder = new TopologyBuilder();
-builder.setSpout("sentences", new KestrelSpout("localhost",22133,"sentence_queue",new StringScheme()));
-builder.setBolt("split", new SplitSentence(), 10)
- .shuffleGrouping("sentences");
-builder.setBolt("count", new WordCount(), 20)
- .fieldsGrouping("split", new Fields("word"));
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">## Execution
+
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">TopologyBuilder builder = new TopologyBuilder();
+builder.setSpout("sentences", new KestrelSpout("localhost",22133,"sentence_queue",new StringScheme()));
+builder.setBolt("split", new SplitSentence(), 10)
+ .shuffleGrouping("sentences");
+builder.setBolt("count", new WordCount(), 20)
+ .fieldsGrouping("split", new Fields("word"));
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">
+## Execution
At first, start your local kestrel server in production or development mode.
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Local-mode.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Local-mode.html b/_site/documentation/Local-mode.html
index feef1cb..6502363 100644
--- a/_site/documentation/Local-mode.html
+++ b/_site/documentation/Local-mode.html
@@ -95,7 +95,7 @@
<p>To create an in-process cluster, simply use the <code>LocalCluster</code> class. For example:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kn">import</span> <span class="nn">backtype.storm.LocalCluster</span><span class="o">;</span>
-<span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">LocalCluster</span><span class="o">();</span>
+<span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LocalCluster</span><span class="o">();</span>
</code></pre></div>
<p>You can then submit topologies using the <code>submitTopology</code> method on the <code>LocalCluster</code> object. Just like the corresponding method on <a href="/javadoc/apidocs/backtype/storm/StormSubmitter.html">StormSubmitter</a>, <code>submitTopology</code> takes a name, a topology configuration, and the topology object. You can then kill a topology using the <code>killTopology</code> method which takes the topology name as an argument.</p>
[03/10] storm git commit: STORM-1366. Add documentation for StormSQL
into the website.
Posted by wh...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/storm-hbase.html
----------------------------------------------------------------------
diff --git a/_site/documentation/storm-hbase.html b/_site/documentation/storm-hbase.html
index 85b58c1..5c638f8 100644
--- a/_site/documentation/storm-hbase.html
+++ b/_site/documentation/storm-hbase.html
@@ -97,9 +97,9 @@
<p>The main API for interacting with HBase is the <code>org.apache.storm.hbase.bolt.mapper.HBaseMapper</code>
interface:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">HBaseMapper</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="kt">byte</span><span class="o">[]</span> <span class="nf">rowKey</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="kt">byte</span><span class="o">[]</span> <span class="n">rowKey</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
- <span class="n">ColumnList</span> <span class="nf">columns</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="n">ColumnList</span> <span class="n">columns</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>The <code>rowKey()</code> method is straightforward: given a Storm tuple, return a byte array representing the
@@ -109,23 +109,23 @@ row key.</p>
to add both standard HBase columns as well as HBase counter columns.</p>
<p>To add a standard column, use one of the <code>addColumn()</code> methods:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ColumnList</span> <span class="n">cols</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">ColumnList</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ColumnList</span> <span class="n">cols</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ColumnList</span><span class="o">();</span>
<span class="n">cols</span><span class="o">.</span><span class="na">addColumn</span><span class="o">(</span><span class="k">this</span><span class="o">.</span><span class="na">columnFamily</span><span class="o">,</span> <span class="n">field</span><span class="o">.</span><span class="na">getBytes</span><span class="o">(),</span> <span class="n">toBytes</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">getValueByField</span><span class="o">(</span><span class="n">field</span><span class="o">)));</span>
</code></pre></div>
<p>To add a counter column, use one of the <code>addCounter()</code> methods:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ColumnList</span> <span class="n">cols</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">ColumnList</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ColumnList</span> <span class="n">cols</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ColumnList</span><span class="o">();</span>
<span class="n">cols</span><span class="o">.</span><span class="na">addCounter</span><span class="o">(</span><span class="k">this</span><span class="o">.</span><span class="na">columnFamily</span><span class="o">,</span> <span class="n">field</span><span class="o">.</span><span class="na">getBytes</span><span class="o">(),</span> <span class="n">toLong</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">getValueByField</span><span class="o">(</span><span class="n">field</span><span class="o">)));</span>
</code></pre></div>
<p>When the remote HBase is security enabled, a kerberos keytab and the corresponding principal name need to be
provided for the storm-hbase connector. Specifically, the Config object passed into the topology should contain
{(“storm.keytab.file”, “$keytab”), ("storm.kerberos.principal", “$principal”)}. Example:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Config</span> <span class="n">config</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Config</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Config</span> <span class="n">config</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Config</span><span class="o">();</span>
<span class="o">...</span>
-<span class="n">config</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"storm.keytab.file"</span><span class="o">,</span> <span class="s">"$keytab"</span><span class="o">);</span>
-<span class="n">config</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"storm.kerberos.principal"</span><span class="o">,</span> <span class="s">"$principle"</span><span class="o">);</span>
-<span class="n">StormSubmitter</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"$topologyName"</span><span class="o">,</span> <span class="n">config</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createTopology</span><span class="o">());</span>
+<span class="n">config</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"storm.keytab.file"</span><span class="o">,</span> <span class="s">"$keytab"</span><span class="o">);</span>
+<span class="n">config</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"storm.kerberos.principal"</span><span class="o">,</span> <span class="s">"$principle"</span><span class="o">);</span>
+<span class="n">StormSubmitter</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"$topologyName"</span><span class="o">,</span> <span class="n">config</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createTopology</span><span class="o">());</span>
</code></pre></div>
-<h2 id="working-with-secure-hbase-using-delegation-tokens.">Working with Secure HBASE using delegation tokens.</h2>
+<h2 id="working-with-secure-hbase-using-delegation-tokens">Working with Secure HBASE using delegation tokens.</h2>
<p>If your topology is going to interact with secure HBase, your bolts/states needs to be authenticated by HBase.
The approach described above requires that all potential worker hosts have "storm.keytab.file" on them. If you have
@@ -176,16 +176,16 @@ tuples to both regular HBase columns as well as counter columns.</p>
<li>Adds an HBase counter column for the tuple field <code>count</code>.</li>
<li>Writes values to the <code>cf</code> column family.</li>
</ol>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">SimpleHBaseMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SimpleHBaseMapper</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withRowKeyField</span><span class="o">(</span><span class="s">"word"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">withCounterFields</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">withColumnFamily</span><span class="o">(</span><span class="s">"cf"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">SimpleHBaseMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SimpleHBaseMapper</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withRowKeyField</span><span class="o">(</span><span class="s">"word"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">withCounterFields</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">withColumnFamily</span><span class="o">(</span><span class="s">"cf"</span><span class="o">);</span>
</code></pre></div>
<h3 id="hbasebolt">HBaseBolt</h3>
<p>To use the <code>HBaseBolt</code>, construct it with the name of the table to write to, an a <code>HBaseMapper</code> implementation:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">HBaseBolt</span> <span class="n">hbase</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HBaseBolt</span><span class="o">(</span><span class="s">"WordCount"</span><span class="o">,</span> <span class="n">mapper</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">HBaseBolt</span> <span class="n">hbase</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HBaseBolt</span><span class="o">(</span><span class="s">"WordCount"</span><span class="o">,</span> <span class="n">mapper</span><span class="o">);</span>
</code></pre></div>
<p>The <code>HBaseBolt</code> will delegate to the <code>mapper</code> instance to figure out how to persist tuple data to HBase.</p>
@@ -193,8 +193,8 @@ tuples to both regular HBase columns as well as counter columns.</p>
<p>This class allows you to transform the HBase lookup result into storm Values that will be emitted by the <code>HBaseLookupBolt</code>.</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">HBaseValueMapper</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">Values</span><span class="o">></span> <span class="nf">toTuples</span><span class="o">(</span><span class="n">Result</span> <span class="n">result</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span><span class="o">;</span>
- <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">Values</span><span class="o">></span> <span class="n">toTuples</span><span class="o">(</span><span class="n">Result</span> <span class="n">result</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span><span class="o">;</span>
+ <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>The <code>toTuples</code> method takes in a HBase <code>Result</code> instance and expects a List of <code>Values</code> instant.
@@ -209,8 +209,8 @@ Each of the value returned by this function will be emitted by the <code>HBaseLo
<p>This class allows you to specify the projection criteria for your HBase Get function. This is optional parameter
for the lookupBolt and if you do not specify this instance all the columns will be returned by <code>HBaseLookupBolt</code>.</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">HBaseProjectionCriteria</span> <span class="kd">implements</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="n">HBaseProjectionCriteria</span> <span class="nf">addColumnFamily</span><span class="o">(</span><span class="n">String</span> <span class="n">columnFamily</span><span class="o">);</span>
- <span class="kd">public</span> <span class="n">HBaseProjectionCriteria</span> <span class="nf">addColumn</span><span class="o">(</span><span class="n">ColumnMetaData</span> <span class="n">column</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="n">HBaseProjectionCriteria</span> <span class="n">addColumnFamily</span><span class="o">(</span><span class="n">String</span> <span class="n">columnFamily</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="n">HBaseProjectionCriteria</span> <span class="n">addColumn</span><span class="o">(</span><span class="n">ColumnMetaData</span> <span class="n">column</span><span class="o">);</span>
</code></pre></div>
<p><code>addColumnFamily</code> takes in columnFamily. Setting this parameter means all columns for this family will be included
in the projection.</p>
@@ -223,9 +223,9 @@ The following code creates a projectionCriteria which specifies a projection cri
<li>includes count column from column family cf.</li>
<li>includes all columns from column family cf2.</li>
</ol>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">HBaseProjectionCriteria</span> <span class="n">projectionCriteria</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HBaseProjectionCriteria</span><span class="o">()</span>
- <span class="o">.</span><span class="na">addColumn</span><span class="o">(</span><span class="k">new</span> <span class="n">HBaseProjectionCriteria</span><span class="o">.</span><span class="na">ColumnMetaData</span><span class="o">(</span><span class="s">"cf"</span><span class="o">,</span> <span class="s">"count"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">addColumnFamily</span><span class="o">(</span><span class="s">"cf2"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">HBaseProjectionCriteria</span> <span class="n">projectionCriteria</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HBaseProjectionCriteria</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">addColumn</span><span class="o">(</span><span class="k">new</span> <span class="n">HBaseProjectionCriteria</span><span class="o">.</span><span class="na">ColumnMetaData</span><span class="o">(</span><span class="s">"cf"</span><span class="o">,</span> <span class="s">"count"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">addColumnFamily</span><span class="o">(</span><span class="s">"cf2"</span><span class="o">);</span>
</code></pre></div>
<h3 id="hbaselookupbolt">HBaseLookupBolt</h3>
@@ -238,7 +238,7 @@ values to be emitted by the bolt.</p>
<p>You can look at an example topology LookupWordCount.java under <code>src/test/java</code>.</p>
-<h2 id="example:-persistent-word-count">Example: Persistent Word Count</h2>
+<h2 id="example-persistent-word-count">Example: Persistent Word Count</h2>
<p>A runnable example can be found in the <code>src/test/java</code> directory.</p>
@@ -248,7 +248,7 @@ values to be emitted by the bolt.</p>
classpath pointing to your HBase cluster.</p>
<p>Use the <code>hbase shell</code> command to create the schema:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">> create 'WordCount', 'cf'
+<div class="highlight"><pre><code class="language-" data-lang="">> create 'WordCount', 'cf'
</code></pre></div>
<h3 id="execution">Execution</h3>
@@ -256,47 +256,47 @@ classpath pointing to your HBase cluster.</p>
<p>After (or while) the word count topology is running, run the <code>org.apache.storm.hbase.topology.WordCountClient</code> class
to view the counter values stored in HBase. You should see something like to following:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">Word: 'apple', Count: 6867
-Word: 'orange', Count: 6645
-Word: 'pineapple', Count: 6954
-Word: 'banana', Count: 6787
-Word: 'watermelon', Count: 6806
+<div class="highlight"><pre><code class="language-" data-lang="">Word: 'apple', Count: 6867
+Word: 'orange', Count: 6645
+Word: 'pineapple', Count: 6954
+Word: 'banana', Count: 6787
+Word: 'watermelon', Count: 6806
</code></pre></div>
<p>For reference, the sample topology is listed below:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">PersistentWordCount</span> <span class="o">{</span>
- <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">WORD_SPOUT</span> <span class="o">=</span> <span class="s">"WORD_SPOUT"</span><span class="o">;</span>
- <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">COUNT_BOLT</span> <span class="o">=</span> <span class="s">"COUNT_BOLT"</span><span class="o">;</span>
- <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">HBASE_BOLT</span> <span class="o">=</span> <span class="s">"HBASE_BOLT"</span><span class="o">;</span>
+ <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">WORD_SPOUT</span> <span class="o">=</span> <span class="s">"WORD_SPOUT"</span><span class="o">;</span>
+ <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">COUNT_BOLT</span> <span class="o">=</span> <span class="s">"COUNT_BOLT"</span><span class="o">;</span>
+ <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">HBASE_BOLT</span> <span class="o">=</span> <span class="s">"HBASE_BOLT"</span><span class="o">;</span>
- <span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span>
- <span class="n">Config</span> <span class="n">config</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Config</span><span class="o">();</span>
+ <span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="n">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span>
+ <span class="n">Config</span> <span class="n">config</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Config</span><span class="o">();</span>
- <span class="n">WordSpout</span> <span class="n">spout</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">WordSpout</span><span class="o">();</span>
- <span class="n">WordCounter</span> <span class="n">bolt</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">WordCounter</span><span class="o">();</span>
+ <span class="n">WordSpout</span> <span class="n">spout</span> <span class="o">=</span> <span class="k">new</span> <span class="n">WordSpout</span><span class="o">();</span>
+ <span class="n">WordCounter</span> <span class="n">bolt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">WordCounter</span><span class="o">();</span>
- <span class="n">SimpleHBaseMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SimpleHBaseMapper</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withRowKeyField</span><span class="o">(</span><span class="s">"word"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">withCounterFields</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">withColumnFamily</span><span class="o">(</span><span class="s">"cf"</span><span class="o">);</span>
+ <span class="n">SimpleHBaseMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SimpleHBaseMapper</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withRowKeyField</span><span class="o">(</span><span class="s">"word"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">withCounterFields</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">withColumnFamily</span><span class="o">(</span><span class="s">"cf"</span><span class="o">);</span>
- <span class="n">HBaseBolt</span> <span class="n">hbase</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HBaseBolt</span><span class="o">(</span><span class="s">"WordCount"</span><span class="o">,</span> <span class="n">mapper</span><span class="o">);</span>
+ <span class="n">HBaseBolt</span> <span class="n">hbase</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HBaseBolt</span><span class="o">(</span><span class="s">"WordCount"</span><span class="o">,</span> <span class="n">mapper</span><span class="o">);</span>
<span class="c1">// wordSpout ==> countBolt ==> HBaseBolt</span>
- <span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TopologyBuilder</span><span class="o">();</span>
+ <span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TopologyBuilder</span><span class="o">();</span>
<span class="n">builder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="n">WORD_SPOUT</span><span class="o">,</span> <span class="n">spout</span><span class="o">,</span> <span class="mi">1</span><span class="o">);</span>
<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="n">COUNT_BOLT</span><span class="o">,</span> <span class="n">bolt</span><span class="o">,</span> <span class="mi">1</span><span class="o">).</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="n">WORD_SPOUT</span><span class="o">);</span>
- <span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="n">HBASE_BOLT</span><span class="o">,</span> <span class="n">hbase</span><span class="o">,</span> <span class="mi">1</span><span class="o">).</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="n">COUNT_BOLT</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
+ <span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="n">HBASE_BOLT</span><span class="o">,</span> <span class="n">hbase</span><span class="o">,</span> <span class="mi">1</span><span class="o">).</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="n">COUNT_BOLT</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span>
<span class="k">if</span> <span class="o">(</span><span class="n">args</span><span class="o">.</span><span class="na">length</span> <span class="o">==</span> <span class="mi">0</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">LocalCluster</span><span class="o">();</span>
- <span class="n">cluster</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"test"</span><span class="o">,</span> <span class="n">config</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createTopology</span><span class="o">());</span>
+ <span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LocalCluster</span><span class="o">();</span>
+ <span class="n">cluster</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"test"</span><span class="o">,</span> <span class="n">config</span><span class="o">,</span> <span class="n">builder</span><span class="o">.</span><span class="na">createTopology</span><span class="o">());</span>
<span class="n">Thread</span><span class="o">.</span><span class="na">sleep</span><span class="o">(</span><span class="mi">10000</span><span class="o">);</span>
- <span class="n">cluster</span><span class="o">.</span><span class="na">killTopology</span><span class="o">(</span><span class="s">"test"</span><span class="o">);</span>
+ <span class="n">cluster</span><span class="o">.</span><span class="na">killTopology</span><span class="o">(</span><span class="s">"test"</span><span class="o">);</span>
<span class="n">cluster</span><span class="o">.</span><span class="na">shutdown</span><span class="o">();</span>
<span class="n">System</span><span class="o">.</span><span class="na">exit</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span>
<span class="o">}</span> <span class="k">else</span> <span class="o">{</span>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/storm-hdfs.html
----------------------------------------------------------------------
diff --git a/_site/documentation/storm-hdfs.html b/_site/documentation/storm-hdfs.html
index 26ccf61..2372d3d 100644
--- a/_site/documentation/storm-hdfs.html
+++ b/_site/documentation/storm-hdfs.html
@@ -97,21 +97,21 @@
<p>The following example will write pipe("|")-delimited files to the HDFS path hdfs://localhost:54310/foo. After every
1,000 tuples it will sync filesystem, making that data visible to other HDFS clients. It will rotate files when they
reach 5 megabytes in size.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="c1">// use "|" instead of "," for field delimiter</span>
-<span class="n">RecordFormat</span> <span class="n">format</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DelimitedRecordFormat</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withFieldDelimiter</span><span class="o">(</span><span class="s">"|"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="c1">// use "|" instead of "," for field delimiter</span>
+<span class="n">RecordFormat</span> <span class="n">format</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DelimitedRecordFormat</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withFieldDelimiter</span><span class="o">(</span><span class="s">"|"</span><span class="o">);</span>
<span class="c1">// sync the filesystem after every 1k tuples</span>
-<span class="n">SyncPolicy</span> <span class="n">syncPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">CountSyncPolicy</span><span class="o">(</span><span class="mi">1000</span><span class="o">);</span>
+<span class="n">SyncPolicy</span> <span class="n">syncPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="n">CountSyncPolicy</span><span class="o">(</span><span class="mi">1000</span><span class="o">);</span>
<span class="c1">// rotate files when they reach 5MB</span>
-<span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
+<span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
-<span class="n">FileNameFormat</span> <span class="n">fileNameFormat</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DefaultFileNameFormat</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withPath</span><span class="o">(</span><span class="s">"/foo/"</span><span class="o">);</span>
+<span class="n">FileNameFormat</span> <span class="n">fileNameFormat</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DefaultFileNameFormat</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withPath</span><span class="o">(</span><span class="s">"/foo/"</span><span class="o">);</span>
-<span class="n">HdfsBolt</span> <span class="n">bolt</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HdfsBolt</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">)</span>
+<span class="n">HdfsBolt</span> <span class="n">bolt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HdfsBolt</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">)</span>
<span class="o">.</span><span class="na">withFileNameFormat</span><span class="o">(</span><span class="n">fileNameFormat</span><span class="o">)</span>
<span class="o">.</span><span class="na">withRecordFormat</span><span class="o">(</span><span class="n">format</span><span class="o">)</span>
<span class="o">.</span><span class="na">withRotationPolicy</span><span class="o">(</span><span class="n">rotationPolicy</span><span class="o">)</span>
@@ -126,7 +126,7 @@ reach 5 megabytes in size.</p>
resolution.</p>
<p>If you experience errors such as the following:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">java.lang.RuntimeException: Error preparing HdfsBolt: No FileSystem for scheme: hdfs
+<div class="highlight"><pre><code class="language-" data-lang="">java.lang.RuntimeException: Error preparing HdfsBolt: No FileSystem for scheme: hdfs
</code></pre></div>
<p>it's an indication that your topology jar file isn't packaged properly.</p>
@@ -148,9 +148,9 @@ create your topology jar:</p>
<span class="nt"><configuration></span>
<span class="nt"><transformers></span>
<span class="nt"><transformer</span>
- <span class="na">implementation=</span><span class="s">"org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"</span><span class="nt">/></span>
+ <span class="na">implementation=</span><span class="s">"org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"</span><span class="nt">/></span>
<span class="nt"><transformer</span>
- <span class="na">implementation=</span><span class="s">"org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"</span><span class="nt">></span>
+ <span class="na">implementation=</span><span class="s">"org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"</span><span class="nt">></span>
<span class="nt"><mainClass></mainClass></span>
<span class="nt"></transformer></span>
<span class="nt"></transformers></span>
@@ -158,6 +158,7 @@ create your topology jar:</p>
<span class="nt"></execution></span>
<span class="nt"></executions></span>
<span class="nt"></plugin></span>
+
</code></pre></div>
<h3 id="specifying-a-hadoop-version">Specifying a Hadoop Version</h3>
@@ -189,7 +190,7 @@ create your topology jar:</p>
and add the dependencies for your preferred version in your pom.</p>
<p>Hadoop client version incompatibilites can manifest as errors like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">com.google.protobuf.InvalidProtocolBufferException: Protocol message contained an invalid tag (zero)
+<div class="highlight"><pre><code class="language-" data-lang="">com.google.protobuf.InvalidProtocolBufferException: Protocol message contained an invalid tag (zero)
</code></pre></div>
<h2 id="customization">Customization</h2>
@@ -198,7 +199,7 @@ and add the dependencies for your preferred version in your pom.</p>
<p>Record format can be controlled by providing an implementation of the <code>org.apache.storm.hdfs.format.RecordFormat</code>
interface:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">RecordFormat</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="kt">byte</span><span class="o">[]</span> <span class="nf">format</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="kt">byte</span><span class="o">[]</span> <span class="n">format</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>The provided <code>org.apache.storm.hdfs.format.DelimitedRecordFormat</code> is capable of producing formats such as CSV and
@@ -209,16 +210,16 @@ tab-delimited files.</p>
<p>File naming can be controlled by providing an implementation of the <code>org.apache.storm.hdfs.format.FileNameFormat</code>
interface:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">FileNameFormat</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="kt">void</span> <span class="nf">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">topologyContext</span><span class="o">);</span>
- <span class="n">String</span> <span class="nf">getName</span><span class="o">(</span><span class="kt">long</span> <span class="n">rotation</span><span class="o">,</span> <span class="kt">long</span> <span class="n">timeStamp</span><span class="o">);</span>
- <span class="n">String</span> <span class="nf">getPath</span><span class="o">();</span>
+ <span class="kt">void</span> <span class="n">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">topologyContext</span><span class="o">);</span>
+ <span class="n">String</span> <span class="n">getName</span><span class="o">(</span><span class="kt">long</span> <span class="n">rotation</span><span class="o">,</span> <span class="kt">long</span> <span class="n">timeStamp</span><span class="o">);</span>
+ <span class="n">String</span> <span class="n">getPath</span><span class="o">();</span>
<span class="o">}</span>
</code></pre></div>
<p>The provided <code>org.apache.storm.hdfs.format.DefaultFileNameFormat</code> will create file names with the following format:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"> {prefix}{componentId}-{taskId}-{rotationNum}-{timestamp}{extension}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="w"> </span><span class="p">{</span><span class="err">prefix</span><span class="p">}{</span><span class="err">componentId</span><span class="p">}</span><span class="err">-</span><span class="p">{</span><span class="err">taskId</span><span class="p">}</span><span class="err">-</span><span class="p">{</span><span class="err">rotationNum</span><span class="p">}</span><span class="err">-</span><span class="p">{</span><span class="err">timestamp</span><span class="p">}{</span><span class="err">extension</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>For example:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"> MyBolt-5-7-1390579837830.txt
+<div class="highlight"><pre><code class="language-" data-lang=""> MyBolt-5-7-1390579837830.txt
</code></pre></div>
<p>By default, prefix is empty and extenstion is ".txt".</p>
@@ -227,8 +228,8 @@ interface:</p>
<p>Sync policies allow you to control when buffered data is flushed to the underlying filesystem (thus making it available
to clients reading the data) by implementing the <code>org.apache.storm.hdfs.sync.SyncPolicy</code> interface:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">SyncPolicy</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="kt">boolean</span> <span class="nf">mark</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="kt">long</span> <span class="n">offset</span><span class="o">);</span>
- <span class="kt">void</span> <span class="nf">reset</span><span class="o">();</span>
+ <span class="kt">boolean</span> <span class="n">mark</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="kt">long</span> <span class="n">offset</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">reset</span><span class="o">();</span>
<span class="o">}</span>
</code></pre></div>
<p>The <code>HdfsBolt</code> will call the <code>mark()</code> method for every tuple it processes. Returning <code>true</code> will trigger the <code>HdfsBolt</code>
@@ -242,13 +243,13 @@ been processed.</p>
<p>Similar to sync policies, file rotation policies allow you to control when data files are rotated by providing a
<code>org.apache.storm.hdfs.rotation.FileRotation</code> interface:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">FileRotationPolicy</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="kt">boolean</span> <span class="nf">mark</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="kt">long</span> <span class="n">offset</span><span class="o">);</span>
- <span class="kt">void</span> <span class="nf">reset</span><span class="o">();</span>
+ <span class="kt">boolean</span> <span class="n">mark</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="kt">long</span> <span class="n">offset</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">reset</span><span class="o">();</span>
<span class="o">}</span>
</code></pre></div>
<p>The <code>org.apache.storm.hdfs.rotation.FileSizeRotationPolicy</code> implementation allows you to trigger file rotation when
data files reach a specific file size:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
</code></pre></div>
<h3 id="file-rotation-actions">File Rotation Actions</h3>
@@ -256,7 +257,7 @@ data files reach a specific file size:</p>
What <code>RotationAction</code>s do is provide a hook to allow you to perform some action right after a file is rotated. For
example, moving a file to a different location or renaming it.</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">RotationAction</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">FileSystem</span> <span class="n">fileSystem</span><span class="o">,</span> <span class="n">Path</span> <span class="n">filePath</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">IOException</span><span class="o">;</span>
+ <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">FileSystem</span> <span class="n">fileSystem</span><span class="o">,</span> <span class="n">Path</span> <span class="n">filePath</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">IOException</span><span class="o">;</span>
<span class="o">}</span>
</code></pre></div>
<p>Storm-HDFS includes a simple action that will move a file after rotation:</p>
@@ -265,15 +266,15 @@ example, moving a file to a different location or renaming it.</p>
<span class="kd">private</span> <span class="n">String</span> <span class="n">destination</span><span class="o">;</span>
- <span class="kd">public</span> <span class="n">MoveFileAction</span> <span class="nf">withDestination</span><span class="o">(</span><span class="n">String</span> <span class="n">destDir</span><span class="o">){</span>
+ <span class="kd">public</span> <span class="n">MoveFileAction</span> <span class="n">withDestination</span><span class="o">(</span><span class="n">String</span> <span class="n">destDir</span><span class="o">){</span>
<span class="n">destination</span> <span class="o">=</span> <span class="n">destDir</span><span class="o">;</span>
<span class="k">return</span> <span class="k">this</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">FileSystem</span> <span class="n">fileSystem</span><span class="o">,</span> <span class="n">Path</span> <span class="n">filePath</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">IOException</span> <span class="o">{</span>
- <span class="n">Path</span> <span class="n">destPath</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Path</span><span class="o">(</span><span class="n">destination</span><span class="o">,</span> <span class="n">filePath</span><span class="o">.</span><span class="na">getName</span><span class="o">());</span>
- <span class="n">LOG</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">"Moving file {} to {}"</span><span class="o">,</span> <span class="n">filePath</span><span class="o">,</span> <span class="n">destPath</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">FileSystem</span> <span class="n">fileSystem</span><span class="o">,</span> <span class="n">Path</span> <span class="n">filePath</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">IOException</span> <span class="o">{</span>
+ <span class="n">Path</span> <span class="n">destPath</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Path</span><span class="o">(</span><span class="n">destination</span><span class="o">,</span> <span class="n">filePath</span><span class="o">.</span><span class="na">getName</span><span class="o">());</span>
+ <span class="n">LOG</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">"Moving file {} to {}"</span><span class="o">,</span> <span class="n">filePath</span><span class="o">,</span> <span class="n">destPath</span><span class="o">);</span>
<span class="kt">boolean</span> <span class="n">success</span> <span class="o">=</span> <span class="n">fileSystem</span><span class="o">.</span><span class="na">rename</span><span class="o">(</span><span class="n">filePath</span><span class="o">,</span> <span class="n">destPath</span><span class="o">);</span>
<span class="k">return</span><span class="o">;</span>
<span class="o">}</span>
@@ -282,80 +283,80 @@ example, moving a file to a different location or renaming it.</p>
<p>If you are using Trident and sequence files you can do something like this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">HdfsState</span><span class="o">.</span><span class="na">Options</span> <span class="n">seqOpts</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HdfsState</span><span class="o">.</span><span class="na">SequenceFileOptions</span><span class="o">()</span>
<span class="o">.</span><span class="na">withFileNameFormat</span><span class="o">(</span><span class="n">fileNameFormat</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withSequenceFormat</span><span class="o">(</span><span class="k">new</span> <span class="nf">DefaultSequenceFormat</span><span class="o">(</span><span class="s">"key"</span><span class="o">,</span> <span class="s">"data"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">withSequenceFormat</span><span class="o">(</span><span class="k">new</span> <span class="n">DefaultSequenceFormat</span><span class="o">(</span><span class="s">"key"</span><span class="o">,</span> <span class="s">"data"</span><span class="o">))</span>
<span class="o">.</span><span class="na">withRotationPolicy</span><span class="o">(</span><span class="n">rotationPolicy</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">addRotationAction</span><span class="o">(</span><span class="k">new</span> <span class="nf">MoveFileAction</span><span class="o">().</span><span class="na">withDestination</span><span class="o">(</span><span class="s">"/dest2/"</span><span class="o">));</span>
+ <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">addRotationAction</span><span class="o">(</span><span class="k">new</span> <span class="n">MoveFileAction</span><span class="o">().</span><span class="na">withDestination</span><span class="o">(</span><span class="s">"/dest2/"</span><span class="o">));</span>
</code></pre></div>
<h2 id="support-for-hdfs-sequence-files">Support for HDFS Sequence Files</h2>
<p>The <code>org.apache.storm.hdfs.bolt.SequenceFileBolt</code> class allows you to write storm data to HDFS sequence files:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="c1">// sync the filesystem after every 1k tuples</span>
- <span class="n">SyncPolicy</span> <span class="n">syncPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">CountSyncPolicy</span><span class="o">(</span><span class="mi">1000</span><span class="o">);</span>
+ <span class="n">SyncPolicy</span> <span class="n">syncPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="n">CountSyncPolicy</span><span class="o">(</span><span class="mi">1000</span><span class="o">);</span>
<span class="c1">// rotate files when they reach 5MB</span>
- <span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
+ <span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
- <span class="n">FileNameFormat</span> <span class="n">fileNameFormat</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DefaultFileNameFormat</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withExtension</span><span class="o">(</span><span class="s">".seq"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withPath</span><span class="o">(</span><span class="s">"/data/"</span><span class="o">);</span>
+ <span class="n">FileNameFormat</span> <span class="n">fileNameFormat</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DefaultFileNameFormat</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withExtension</span><span class="o">(</span><span class="s">".seq"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withPath</span><span class="o">(</span><span class="s">"/data/"</span><span class="o">);</span>
<span class="c1">// create sequence format instance.</span>
- <span class="n">DefaultSequenceFormat</span> <span class="n">format</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DefaultSequenceFormat</span><span class="o">(</span><span class="s">"timestamp"</span><span class="o">,</span> <span class="s">"sentence"</span><span class="o">);</span>
+ <span class="n">DefaultSequenceFormat</span> <span class="n">format</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DefaultSequenceFormat</span><span class="o">(</span><span class="s">"timestamp"</span><span class="o">,</span> <span class="s">"sentence"</span><span class="o">);</span>
- <span class="n">SequenceFileBolt</span> <span class="n">bolt</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SequenceFileBolt</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">)</span>
+ <span class="n">SequenceFileBolt</span> <span class="n">bolt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SequenceFileBolt</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">)</span>
<span class="o">.</span><span class="na">withFileNameFormat</span><span class="o">(</span><span class="n">fileNameFormat</span><span class="o">)</span>
<span class="o">.</span><span class="na">withSequenceFormat</span><span class="o">(</span><span class="n">format</span><span class="o">)</span>
<span class="o">.</span><span class="na">withRotationPolicy</span><span class="o">(</span><span class="n">rotationPolicy</span><span class="o">)</span>
<span class="o">.</span><span class="na">withSyncPolicy</span><span class="o">(</span><span class="n">syncPolicy</span><span class="o">)</span>
<span class="o">.</span><span class="na">withCompressionType</span><span class="o">(</span><span class="n">SequenceFile</span><span class="o">.</span><span class="na">CompressionType</span><span class="o">.</span><span class="na">RECORD</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withCompressionCodec</span><span class="o">(</span><span class="s">"deflate"</span><span class="o">);</span>
+ <span class="o">.</span><span class="na">withCompressionCodec</span><span class="o">(</span><span class="s">"deflate"</span><span class="o">);</span>
</code></pre></div>
<p>The <code>SequenceFileBolt</code> requires that you provide a <code>org.apache.storm.hdfs.bolt.format.SequenceFormat</code> that maps tuples to
key/value pairs:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">SequenceFormat</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="n">Class</span> <span class="nf">keyClass</span><span class="o">();</span>
- <span class="n">Class</span> <span class="nf">valueClass</span><span class="o">();</span>
+ <span class="n">Class</span> <span class="n">keyClass</span><span class="o">();</span>
+ <span class="n">Class</span> <span class="n">valueClass</span><span class="o">();</span>
- <span class="n">Writable</span> <span class="nf">key</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
- <span class="n">Writable</span> <span class="nf">value</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="n">Writable</span> <span class="n">key</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="n">Writable</span> <span class="n">value</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<h2 id="trident-api">Trident API</h2>
<p>storm-hdfs also includes a Trident <code>state</code> implementation for writing data to HDFS, with an API that closely mirrors
that of the bolts.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">Fields</span> <span class="n">hdfsFields</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"field1"</span><span class="o">,</span> <span class="s">"field2"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">Fields</span> <span class="n">hdfsFields</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"field1"</span><span class="o">,</span> <span class="s">"field2"</span><span class="o">);</span>
- <span class="n">FileNameFormat</span> <span class="n">fileNameFormat</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DefaultFileNameFormat</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withPath</span><span class="o">(</span><span class="s">"/trident"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withPrefix</span><span class="o">(</span><span class="s">"trident"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withExtension</span><span class="o">(</span><span class="s">".txt"</span><span class="o">);</span>
+ <span class="n">FileNameFormat</span> <span class="n">fileNameFormat</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DefaultFileNameFormat</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withPath</span><span class="o">(</span><span class="s">"/trident"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withPrefix</span><span class="o">(</span><span class="s">"trident"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withExtension</span><span class="o">(</span><span class="s">".txt"</span><span class="o">);</span>
- <span class="n">RecordFormat</span> <span class="n">recordFormat</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DelimitedRecordFormat</span><span class="o">()</span>
+ <span class="n">RecordFormat</span> <span class="n">recordFormat</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DelimitedRecordFormat</span><span class="o">()</span>
<span class="o">.</span><span class="na">withFields</span><span class="o">(</span><span class="n">hdfsFields</span><span class="o">);</span>
- <span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">FileSizeRotationPolicy</span><span class="o">.</span><span class="na">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
+ <span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">FileSizeRotationPolicy</span><span class="o">.</span><span class="na">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
<span class="n">HdfsState</span><span class="o">.</span><span class="na">Options</span> <span class="n">options</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HdfsState</span><span class="o">.</span><span class="na">HdfsFileOptions</span><span class="o">()</span>
<span class="o">.</span><span class="na">withFileNameFormat</span><span class="o">(</span><span class="n">fileNameFormat</span><span class="o">)</span>
<span class="o">.</span><span class="na">withRecordFormat</span><span class="o">(</span><span class="n">recordFormat</span><span class="o">)</span>
<span class="o">.</span><span class="na">withRotationPolicy</span><span class="o">(</span><span class="n">rotationPolicy</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">);</span>
+ <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">);</span>
- <span class="n">StateFactory</span> <span class="n">factory</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HdfsStateFactory</span><span class="o">().</span><span class="na">withOptions</span><span class="o">(</span><span class="n">options</span><span class="o">);</span>
+ <span class="n">StateFactory</span> <span class="n">factory</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HdfsStateFactory</span><span class="o">().</span><span class="na">withOptions</span><span class="o">(</span><span class="n">options</span><span class="o">);</span>
<span class="n">TridentState</span> <span class="n">state</span> <span class="o">=</span> <span class="n">stream</span>
- <span class="o">.</span><span class="na">partitionPersist</span><span class="o">(</span><span class="n">factory</span><span class="o">,</span> <span class="n">hdfsFields</span><span class="o">,</span> <span class="k">new</span> <span class="nf">HdfsUpdater</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">());</span>
+ <span class="o">.</span><span class="na">partitionPersist</span><span class="o">(</span><span class="n">factory</span><span class="o">,</span> <span class="n">hdfsFields</span><span class="o">,</span> <span class="k">new</span> <span class="n">HdfsUpdater</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">());</span>
</code></pre></div>
<p>To use the sequence file <code>State</code> implementation, use the <code>HdfsState.SequenceFileOptions</code>:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">HdfsState</span><span class="o">.</span><span class="na">Options</span> <span class="n">seqOpts</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HdfsState</span><span class="o">.</span><span class="na">SequenceFileOptions</span><span class="o">()</span>
<span class="o">.</span><span class="na">withFileNameFormat</span><span class="o">(</span><span class="n">fileNameFormat</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withSequenceFormat</span><span class="o">(</span><span class="k">new</span> <span class="nf">DefaultSequenceFormat</span><span class="o">(</span><span class="s">"key"</span><span class="o">,</span> <span class="s">"data"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">withSequenceFormat</span><span class="o">(</span><span class="k">new</span> <span class="n">DefaultSequenceFormat</span><span class="o">(</span><span class="s">"key"</span><span class="o">,</span> <span class="s">"data"</span><span class="o">))</span>
<span class="o">.</span><span class="na">withRotationPolicy</span><span class="o">(</span><span class="n">rotationPolicy</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">addRotationAction</span><span class="o">(</span><span class="k">new</span> <span class="nf">MoveFileAction</span><span class="o">().</span><span class="na">toDestination</span><span class="o">(</span><span class="s">"/dest2/"</span><span class="o">));</span>
+ <span class="o">.</span><span class="na">withFsUrl</span><span class="o">(</span><span class="s">"hdfs://localhost:54310"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">addRotationAction</span><span class="o">(</span><span class="k">new</span> <span class="n">MoveFileAction</span><span class="o">().</span><span class="na">toDestination</span><span class="o">(</span><span class="s">"/dest2/"</span><span class="o">));</span>
</code></pre></div>
<h2 id="working-with-secure-hdfs">Working with Secure HDFS</h2>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/storm-hive.html
----------------------------------------------------------------------
diff --git a/_site/documentation/storm-hive.html b/_site/documentation/storm-hive.html
index b99ac62..8db4dff 100644
--- a/_site/documentation/storm-hive.html
+++ b/_site/documentation/storm-hive.html
@@ -97,18 +97,18 @@
<p>With the help of Hive Streaming API, HiveBolt and HiveState allows users to stream data from Storm into Hive directly.
To use Hive streaming API users need to create a bucketed table with ORC format. Example below</p>
-<div class="highlight"><pre><code class="language-sql" data-lang="sql"> <span class="k">create</span> <span class="k">table</span> <span class="n">test_table</span> <span class="p">(</span> <span class="n">id</span> <span class="nb">INT</span><span class="p">,</span> <span class="n">name</span> <span class="n">STRING</span><span class="p">,</span> <span class="n">phone</span> <span class="n">STRING</span><span class="p">,</span> <span class="n">street</span> <span class="n">STRING</span><span class="p">)</span> <span class="n">partitioned</span> <span class="k">by</span> <span class="p">(</span><span class="n">city</span> <span class="n">STRING</span><span class="p">,</span> <span class="k">state</span> <span class="n">STRING</span><span class="p">)</span> <span class="n">stored</span> <span class="k">as</span> <span class="n">orc</span> <span class="n">tblproperties</span> <span class="p">(</span><span class="ss">"orc.compress"</span><span class="o">=</span><span class="
ss">"NONE"</span><span class="p">);</span>
+<div class="highlight"><pre><code class="language-sql" data-lang="sql"> <span class="k">create</span> <span class="k">table</span> <span class="n">test_table</span> <span class="p">(</span> <span class="n">id</span> <span class="n">INT</span><span class="p">,</span> <span class="n">name</span> <span class="n">STRING</span><span class="p">,</span> <span class="n">phone</span> <span class="n">STRING</span><span class="p">,</span> <span class="n">street</span> <span class="n">STRING</span><span class="p">)</span> <span class="n">partitioned</span> <span class="k">by</span> <span class="p">(</span><span class="n">city</span> <span class="n">STRING</span><span class="p">,</span> <span class="k">state</span> <span class="n">STRING</span><span class="p">)</span> <span class="n">stored</span> <span class="k">as</span> <span class="n">orc</span> <span class="n">tblproperties</span> <span class="p">(</span><span class="nv">"orc.compress"</span><span class="o">=</span><span class="nv">"NONE"<
/span><span class="p">);</span>
</code></pre></div>
-<h2 id="hivebolt-(org.apache.storm.hive.bolt.hivebolt)">HiveBolt (org.apache.storm.hive.bolt.HiveBolt)</h2>
+<h2 id="hivebolt-org-apache-storm-hive-bolt-hivebolt">HiveBolt (org.apache.storm.hive.bolt.HiveBolt)</h2>
<p>HiveBolt streams tuples directly into Hive. Tuples are written using Hive Transactions.
Partitions to which HiveBolt will stream to can either created or pre-created or optionally
HiveBolt can create them if they are missing. Fields from Tuples are mapped to table columns.
User should make sure that Tuple field names are matched to the table column names.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">DelimitedRecordHiveMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DelimitedRecordHiveMapper</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="n">colNames</span><span class="o">));</span>
-<span class="n">HiveOptions</span> <span class="n">hiveOptions</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HiveOptions</span><span class="o">(</span><span class="n">metaStoreURI</span><span class="o">,</span><span class="n">dbName</span><span class="o">,</span><span class="n">tblName</span><span class="o">,</span><span class="n">mapper</span><span class="o">);</span>
-<span class="n">HiveBolt</span> <span class="n">hiveBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HiveBolt</span><span class="o">(</span><span class="n">hiveOptions</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">DelimitedRecordHiveMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DelimitedRecordHiveMapper</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="n">colNames</span><span class="o">));</span>
+<span class="n">HiveOptions</span> <span class="n">hiveOptions</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HiveOptions</span><span class="o">(</span><span class="n">metaStoreURI</span><span class="o">,</span><span class="n">dbName</span><span class="o">,</span><span class="n">tblName</span><span class="o">,</span><span class="n">mapper</span><span class="o">);</span>
+<span class="n">HiveBolt</span> <span class="n">hiveBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HiveBolt</span><span class="o">(</span><span class="n">hiveOptions</span><span class="o">);</span>
</code></pre></div>
<h3 id="recordhivemapper">RecordHiveMapper</h3>
@@ -119,13 +119,13 @@ User should make sure that Tuple field names are matched to the table column nam
<li>DelimitedRecordHiveMapper (org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper)</li>
<li>JsonRecordHiveMapper (org.apache.storm.hive.bolt.mapper.JsonRecordHiveMapper)</li>
</ul>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">DelimitedRecordHiveMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DelimitedRecordHiveMapper</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="n">colNames</span><span class="o">))</span>
- <span class="o">.</span><span class="na">withPartitionFields</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="n">partNames</span><span class="o">));</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">DelimitedRecordHiveMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DelimitedRecordHiveMapper</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="n">colNames</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">withPartitionFields</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="n">partNames</span><span class="o">));</span>
<span class="n">or</span>
- <span class="n">DelimitedRecordHiveMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DelimitedRecordHiveMapper</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="n">colNames</span><span class="o">))</span>
- <span class="o">.</span><span class="na">withTimeAsPartitionField</span><span class="o">(</span><span class="s">"YYYY/MM/DD"</span><span class="o">);</span>
+ <span class="n">DelimitedRecordHiveMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DelimitedRecordHiveMapper</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="n">colNames</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">withTimeAsPartitionField</span><span class="o">(</span><span class="s">"YYYY/MM/DD"</span><span class="o">);</span>
</code></pre></div>
<table><thead>
<tr>
@@ -151,10 +151,10 @@ User should make sure that Tuple field names are matched to the table column nam
</tr>
</tbody></table>
-<h3 id="hiveoptions-(org.apache.storm.hive.common.hiveoptions)">HiveOptions (org.apache.storm.hive.common.HiveOptions)</h3>
+<h3 id="hiveoptions-org-apache-storm-hive-common-hiveoptions">HiveOptions (org.apache.storm.hive.common.HiveOptions)</h3>
<p>HiveBolt takes in HiveOptions as a constructor arg.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">HiveOptions</span> <span class="n">hiveOptions</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HiveOptions</span><span class="o">(</span><span class="n">metaStoreURI</span><span class="o">,</span><span class="n">dbName</span><span class="o">,</span><span class="n">tblName</span><span class="o">,</span><span class="n">mapper</span><span class="o">)</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">HiveOptions</span> <span class="n">hiveOptions</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HiveOptions</span><span class="o">(</span><span class="n">metaStoreURI</span><span class="o">,</span><span class="n">dbName</span><span class="o">,</span><span class="n">tblName</span><span class="o">,</span><span class="n">mapper</span><span class="o">)</span>
<span class="o">.</span><span class="na">withTxnsPerBatch</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
<span class="o">.</span><span class="na">withBatchSize</span><span class="o">(</span><span class="mi">1000</span><span class="o">)</span>
<span class="o">.</span><span class="na">withIdleTimeout</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
@@ -235,20 +235,20 @@ User should make sure that Tuple field names are matched to the table column nam
</tr>
</tbody></table>
-<h2 id="hivestate-(org.apache.storm.hive.trident.hivetrident)">HiveState (org.apache.storm.hive.trident.HiveTrident)</h2>
+<h2 id="hivestate-org-apache-storm-hive-trident-hivetrident">HiveState (org.apache.storm.hive.trident.HiveTrident)</h2>
<p>Hive Trident state also follows similar pattern to HiveBolt it takes in HiveOptions as an arg.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">DelimitedRecordHiveMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DelimitedRecordHiveMapper</span><span class="o">()</span>
- <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="n">colNames</span><span class="o">))</span>
- <span class="o">.</span><span class="na">withTimeAsPartitionField</span><span class="o">(</span><span class="s">"YYYY/MM/DD"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">DelimitedRecordHiveMapper</span> <span class="n">mapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DelimitedRecordHiveMapper</span><span class="o">()</span>
+ <span class="o">.</span><span class="na">withColumnFields</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="n">colNames</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">withTimeAsPartitionField</span><span class="o">(</span><span class="s">"YYYY/MM/DD"</span><span class="o">);</span>
- <span class="n">HiveOptions</span> <span class="n">hiveOptions</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HiveOptions</span><span class="o">(</span><span class="n">metaStoreURI</span><span class="o">,</span><span class="n">dbName</span><span class="o">,</span><span class="n">tblName</span><span class="o">,</span><span class="n">mapper</span><span class="o">)</span>
+ <span class="n">HiveOptions</span> <span class="n">hiveOptions</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HiveOptions</span><span class="o">(</span><span class="n">metaStoreURI</span><span class="o">,</span><span class="n">dbName</span><span class="o">,</span><span class="n">tblName</span><span class="o">,</span><span class="n">mapper</span><span class="o">)</span>
<span class="o">.</span><span class="na">withTxnsPerBatch</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
<span class="o">.</span><span class="na">withBatchSize</span><span class="o">(</span><span class="mi">1000</span><span class="o">)</span>
<span class="o">.</span><span class="na">withIdleTimeout</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
- <span class="n">StateFactory</span> <span class="n">factory</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HiveStateFactory</span><span class="o">().</span><span class="na">withOptions</span><span class="o">(</span><span class="n">hiveOptions</span><span class="o">);</span>
- <span class="n">TridentState</span> <span class="n">state</span> <span class="o">=</span> <span class="n">stream</span><span class="o">.</span><span class="na">partitionPersist</span><span class="o">(</span><span class="n">factory</span><span class="o">,</span> <span class="n">hiveFields</span><span class="o">,</span> <span class="k">new</span> <span class="nf">HiveUpdater</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">());</span>
+ <span class="n">StateFactory</span> <span class="n">factory</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HiveStateFactory</span><span class="o">().</span><span class="na">withOptions</span><span class="o">(</span><span class="n">hiveOptions</span><span class="o">);</span>
+ <span class="n">TridentState</span> <span class="n">state</span> <span class="o">=</span> <span class="n">stream</span><span class="o">.</span><span class="na">partitionPersist</span><span class="o">(</span><span class="n">factory</span><span class="o">,</span> <span class="n">hiveFields</span><span class="o">,</span> <span class="k">new</span> <span class="n">HiveUpdater</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">());</span>
</code></pre></div>
[10/10] storm git commit: Merge branch 'asf-site' of
https://github.com/haohui/storm into HEAD
Posted by wh...@apache.org.
Merge branch 'asf-site' of https://github.com/haohui/storm into HEAD
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/573bdcf1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/573bdcf1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/573bdcf1
Branch: refs/heads/asf-site
Commit: 573bdcf1d8ed506cece11fd5f6cc63a4adc175dd
Parents: b962d32 dfa66d7
Author: Haohui Mai <ha...@uber.com>
Authored: Tue Jan 12 14:12:41 2016 -0800
Committer: Haohui Mai <ha...@uber.com>
Committed: Tue Jan 12 14:12:41 2016 -0800
----------------------------------------------------------------------
_site/2012/08/02/storm080-released.html | 2 +-
_site/2012/09/06/storm081-released.html | 2 +-
_site/2013/12/08/storm090-released.html | 4 +-
_site/2014/06/25/storm092-released.html | 2 +-
_site/2015/06/15/storm0100-beta-released.html | 4 +-
_site/contribute/BYLAWS.html | 8 +-
_site/contribute/Contributing-to-Storm.html | 1 +
_site/doc-index.html | 8 +
_site/documentation.html | 5 +-
.../Acking-framework-implementation.html | 2 +-
_site/documentation/Clojure-DSL.html | 182 +++----
_site/documentation/Common-patterns.html | 36 +-
.../Creating-a-new-Storm-project.html | 2 +-
...fining-a-non-jvm-language-dsl-for-storm.html | 8 +-
_site/documentation/Distributed-RPC.html | 62 +--
_site/documentation/FAQ.html | 30 +-
_site/documentation/Fault-tolerance.html | 10 +-
.../Guaranteeing-message-processing.html | 64 +--
_site/documentation/Home.html | 4 +-
.../Installing-native-dependencies.html | 4 +-
_site/documentation/Kestrel-and-Storm.html | 65 +--
_site/documentation/Local-mode.html | 2 +-
_site/documentation/Multilang-protocol.html | 236 ++++-----
...ning-topologies-on-a-production-cluster.html | 4 +-
.../Serialization-(prior-to-0.6.0).html | 6 +-
_site/documentation/Serialization.html | 8 +-
.../Setting-up-a-Storm-cluster.html | 26 +-
.../Setting-up-development-environment.html | 6 +-
...age-protocol-(versions-0.7.0-and-below).html | 86 ++--
.../Structure-of-the-codebase.html | 4 +-
.../documentation/Transactional-topologies.html | 44 +-
_site/documentation/Trident-API-Overview.html | 84 +--
_site/documentation/Trident-spouts.html | 4 +-
_site/documentation/Trident-state.html | 130 ++---
_site/documentation/Trident-tutorial.html | 84 +--
_site/documentation/Troubleshooting.html | 10 +-
...ing-the-parallelism-of-a-Storm-topology.html | 28 +-
.../Using-non-JVM-languages-with-Storm.html | 8 +-
_site/documentation/flux.html | 510 ++++++++++---------
_site/documentation/getting-help.html | 4 +-
_site/documentation/storm-eventhubs.html | 8 +-
_site/documentation/storm-hbase.html | 94 ++--
_site/documentation/storm-hdfs.html | 119 ++---
_site/documentation/storm-hive.html | 42 +-
_site/documentation/storm-jdbc.html | 112 ++--
_site/documentation/storm-kafka.html | 93 ++--
_site/documentation/storm-redis.html | 106 ++--
_site/documentation/storm-solr.html | 32 +-
_site/feed.xml | 32 +-
_site/getting-help.html | 4 +-
_site/tutorial.html | 92 ++--
documentation.html | 5 +-
.../images/storm-sql-internal-example.png | Bin 0 -> 28377 bytes
.../images/storm-sql-internal-workflow.png | Bin 0 -> 20020 bytes
documentation/storm-sql-internal.md | 55 ++
documentation/storm-sql.md | 97 ++++
56 files changed, 1432 insertions(+), 1248 deletions(-)
----------------------------------------------------------------------
[05/10] storm git commit: STORM-1366. Add documentation for StormSQL
into the website.
Posted by wh...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Trident-tutorial.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Trident-tutorial.html b/_site/documentation/Trident-tutorial.html
index ef7ea78..a0f3a1b 100644
--- a/_site/documentation/Trident-tutorial.html
+++ b/_site/documentation/Trident-tutorial.html
@@ -102,20 +102,20 @@
</ol>
<p>For the purposes of illustration, this example will read an infinite stream of sentences from the following source:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">FixedBatchSpout</span> <span class="n">spout</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">FixedBatchSpout</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"sentence"</span><span class="o">),</span> <span class="mi">3</span><span class="o">,</span>
- <span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="s">"the cow jumped over the moon"</span><span class="o">),</span>
- <span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="s">"the man went to the store and bought some candy"</span><span class="o">),</span>
- <span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="s">"four score and seven years ago"</span><span class="o">),</span>
- <span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="s">"how many apples can you eat"</span><span class="o">));</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">FixedBatchSpout</span> <span class="n">spout</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FixedBatchSpout</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"sentence"</span><span class="o">),</span> <span class="mi">3</span><span class="o">,</span>
+ <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="s">"the cow jumped over the moon"</span><span class="o">),</span>
+ <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="s">"the man went to the store and bought some candy"</span><span class="o">),</span>
+ <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="s">"four score and seven years ago"</span><span class="o">),</span>
+ <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="s">"how many apples can you eat"</span><span class="o">));</span>
<span class="n">spout</span><span class="o">.</span><span class="na">setCycle</span><span class="o">(</span><span class="kc">true</span><span class="o">);</span>
</code></pre></div>
<p>This spout cycles through that set of sentences over and over to produce the sentence stream. Here's the code to do the streaming word count part of the computation:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TridentTopology</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TridentTopology</span><span class="o">();</span>
<span class="n">TridentState</span> <span class="n">wordCounts</span> <span class="o">=</span>
- <span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"spout1"</span><span class="o">,</span> <span class="n">spout</span><span class="o">)</span>
- <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"sentence"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Split</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">persistentAggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">MemoryMapState</span><span class="o">.</span><span class="na">Factory</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
+ <span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"spout1"</span><span class="o">,</span> <span class="n">spout</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"sentence"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Split</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">persistentAggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">MemoryMapState</span><span class="o">.</span><span class="na">Factory</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
<span class="o">.</span><span class="na">parallelismHint</span><span class="o">(</span><span class="mi">6</span><span class="o">);</span>
</code></pre></div>
<p>Let's go through the code line by line. First a TridentTopology object is created, which exposes the interface for constructing Trident computations. TridentTopology has a method called newStream that creates a new stream of data in the topology reading from an input source. In this case, the input source is just the FixedBatchSpout defined from before. Input sources can also be queue brokers like Kestrel or Kafka. Trident keeps track of a small amount of state for each input source (metadata about what it has consumed) in Zookeeper, and the "spout1" string here specifies the node in Zookeeper where Trident should keep that metadata.</p>
@@ -130,10 +130,10 @@
<p>Back to the example, the spout emits a stream containing one field called "sentence". The next line of the topology definition applies the Split function to each tuple in the stream, taking the "sentence" field and splitting it into words. Each sentence tuple creates potentially many word tuples – for instance, the sentence "the cow jumped over the moon" creates six "word" tuples. Here's the definition of Split:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">Split</span> <span class="kd">extends</span> <span class="n">BaseFunction</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="n">String</span> <span class="n">sentence</span> <span class="o">=</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span>
- <span class="k">for</span><span class="o">(</span><span class="n">String</span> <span class="nl">word:</span> <span class="n">sentence</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">))</span> <span class="o">{</span>
- <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
+ <span class="k">for</span><span class="o">(</span><span class="n">String</span> <span class="nl">word:</span> <span class="n">sentence</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">))</span> <span class="o">{</span>
+ <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
<span class="o">}</span>
@@ -141,7 +141,7 @@
<p>As you can see, it's really simple. It simply grabs the sentence, splits it on whitespace, and emits a tuple for each word.</p>
<p>The rest of the topology computes word count and keeps the results persistently stored. First the stream is grouped by the "word" field. Then, each group is persistently aggregated using the Count aggregator. The persistentAggregate function knows how to store and update the results of the aggregation in a source of state. In this example, the word counts are kept in memory, but this can be trivially swapped to use Memcached, Cassandra, or any other persistent store. Swapping this topology to store counts in Memcached is as simple as replacing the persistentAggregate line with this (using <a href="https://github.com/nathanmarz/trident-memcached">trident-memcached</a>), where the "serverLocations" is a list of host/ports for the Memcached cluster:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="o">.</span><span class="na">persistentAggregate</span><span class="o">(</span><span class="n">MemcachedState</span><span class="o">.</span><span class="na">transactional</span><span class="o">(</span><span class="n">serverLocations</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="o">.</span><span class="na">persistentAggregate</span><span class="o">(</span><span class="n">MemcachedState</span><span class="o">.</span><span class="na">transactional</span><span class="o">(</span><span class="n">serverLocations</span><span class="o">),</span> <span class="k">new</span> <span class="n">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
<span class="n">MemcachedState</span><span class="o">.</span><span class="na">transactional</span><span class="o">()</span>
</code></pre></div>
<p>The values stored by persistentAggregate represents the aggregation of all batches ever emitted by the stream.</p>
@@ -151,19 +151,19 @@
<p>The persistentAggregate method transforms a Stream into a TridentState object. In this case the TridentState object represents all the word counts. We will use this TridentState object to implement the distributed query portion of the computation.</p>
<p>The next part of the topology implements a low latency distributed query on the word counts. The query takes as input a whitespace separated list of words and return the sum of the counts for those words. These queries are executed just like normal RPC calls, except they are parallelized in the background. Here's an example of how you might invoke one of these queries:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">DRPCClient</span> <span class="n">client</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">DRPCClient</span><span class="o">(</span><span class="s">"drpc.server.location"</span><span class="o">,</span> <span class="mi">3772</span><span class="o">);</span>
-<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">client</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"words"</span><span class="o">,</span> <span class="s">"cat dog the man"</span><span class="o">);</span>
-<span class="c1">// prints the JSON-encoded result, e.g.: "[[5078]]"</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">DRPCClient</span> <span class="n">client</span> <span class="o">=</span> <span class="k">new</span> <span class="n">DRPCClient</span><span class="o">(</span><span class="s">"drpc.server.location"</span><span class="o">,</span> <span class="mi">3772</span><span class="o">);</span>
+<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">client</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"words"</span><span class="o">,</span> <span class="s">"cat dog the man"</span><span class="o">);</span>
+<span class="c1">// prints the JSON-encoded result, e.g.: "[[5078]]"</span>
</code></pre></div>
<p>As you can see, it looks just like a regular remote procedure call (RPC), except it's executing in parallel across a Storm cluster. The latency for small queries like this are typically around 10ms. More intense DRPC queries can take longer of course, although the latency largely depends on how many resources you have allocated for the computation.</p>
<p>The implementation of the distributed query portion of the topology looks like this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">topology</span><span class="o">.</span><span class="na">newDRPCStream</span><span class="o">(</span><span class="s">"words"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"args"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Split</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">stateQuery</span><span class="o">(</span><span class="n">wordCounts</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">MapGet</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">FilterNull</span><span class="o">())</span>
- <span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">));</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">topology</span><span class="o">.</span><span class="na">newDRPCStream</span><span class="o">(</span><span class="s">"words"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"args"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Split</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">stateQuery</span><span class="o">(</span><span class="n">wordCounts</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">),</span> <span class="k">new</span> <span class="n">MapGet</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">),</span> <span class="k">new</span> <span class="n">FilterNull</span><span class="o">())</span>
+ <span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">));</span>
</code></pre></div>
<p>The same TridentTopology object is used to create the DRPC stream, and the function is named "words". The function name corresponds to the function name given in the first argument of execute when using a DRPCClient.</p>
@@ -192,17 +192,17 @@
<span class="n">TridentState</span> <span class="n">tweetersToFollowers</span> <span class="o">=</span>
<span class="n">topology</span><span class="o">.</span><span class="na">newStaticState</span><span class="o">(</span><span class="n">getTweeterToFollowersState</span><span class="o">());</span>
-<span class="n">topology</span><span class="o">.</span><span class="na">newDRPCStream</span><span class="o">(</span><span class="s">"reach"</span><span class="o">)</span>
- <span class="o">.</span><span class="na">stateQuery</span><span class="o">(</span><span class="n">urlToTweeters</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"args"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">MapGet</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"tweeters"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"tweeters"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">ExpandList</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"tweeter"</span><span class="o">))</span>
+<span class="n">topology</span><span class="o">.</span><span class="na">newDRPCStream</span><span class="o">(</span><span class="s">"reach"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">stateQuery</span><span class="o">(</span><span class="n">urlToTweeters</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"args"</span><span class="o">),</span> <span class="k">new</span> <span class="n">MapGet</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"tweeters"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"tweeters"</span><span class="o">),</span> <span class="k">new</span> <span class="n">ExpandList</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"tweeter"</span><span class="o">))</span>
<span class="o">.</span><span class="na">shuffle</span><span class="o">()</span>
- <span class="o">.</span><span class="na">stateQuery</span><span class="o">(</span><span class="n">tweetersToFollowers</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"tweeter"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">MapGet</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"followers"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">stateQuery</span><span class="o">(</span><span class="n">tweetersToFollowers</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"tweeter"</span><span class="o">),</span> <span class="k">new</span> <span class="n">MapGet</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"followers"</span><span class="o">))</span>
<span class="o">.</span><span class="na">parallelismHint</span><span class="o">(</span><span class="mi">200</span><span class="o">)</span>
- <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"followers"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">ExpandList</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"follower"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"follower"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="nf">One</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"one"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"followers"</span><span class="o">),</span> <span class="k">new</span> <span class="n">ExpandList</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"follower"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"follower"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">One</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"one"</span><span class="o">))</span>
<span class="o">.</span><span class="na">parallelismHint</span><span class="o">(</span><span class="mi">20</span><span class="o">)</span>
- <span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="nf">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"reach"</span><span class="o">));</span>
+ <span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"reach"</span><span class="o">));</span>
</code></pre></div>
<p>The topology creates TridentState objects representing each external database using the newStaticState method. These can then be queried in the topology. Like all sources of state, queries to these databases will be automatically batched for maximum efficiency.</p>
@@ -212,15 +212,15 @@
<p>Next, the set of followers is uniqued and counted. This is done in two steps. First a "group by" is done on the batch by "follower", running the "One" aggregator on each group. The "One" aggregator simply emits a single tuple containing the number one for each group. Then, the ones are summed together to get the unique count of the followers set. Here's the definition of the "One" aggregator:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">One</span> <span class="kd">implements</span> <span class="n">CombinerAggregator</span><span class="o"><</span><span class="n">Integer</span><span class="o">></span> <span class="o">{</span>
- <span class="kd">public</span> <span class="n">Integer</span> <span class="nf">init</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">Integer</span> <span class="n">init</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="k">return</span> <span class="mi">1</span><span class="o">;</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="n">Integer</span> <span class="nf">combine</span><span class="o">(</span><span class="n">Integer</span> <span class="n">val1</span><span class="o">,</span> <span class="n">Integer</span> <span class="n">val2</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">Integer</span> <span class="n">combine</span><span class="o">(</span><span class="n">Integer</span> <span class="n">val1</span><span class="o">,</span> <span class="n">Integer</span> <span class="n">val2</span><span class="o">)</span> <span class="o">{</span>
<span class="k">return</span> <span class="mi">1</span><span class="o">;</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="n">Integer</span> <span class="nf">zero</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">Integer</span> <span class="n">zero</span><span class="o">()</span> <span class="o">{</span>
<span class="k">return</span> <span class="mi">1</span><span class="o">;</span>
<span class="o">}</span>
<span class="o">}</span>
@@ -234,11 +234,11 @@
<p>The Trident data model is the TridentTuple which is a named list of values. During a topology, tuples are incrementally built up through a sequence of operations. Operations generally take in a set of input fields and emit a set of "function fields". The input fields are used to select a subset of the tuple as input to the operation, while the "function fields" name the fields the operation emits.</p>
<p>Consider this example. Suppose you have a stream called "stream" that contains the fields "x", "y", and "z". To run a filter MyFilter that takes in "y" as input, you would say:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">stream</span><span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"y"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">MyFilter</span><span class="o">())</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">stream</span><span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"y"</span><span class="o">),</span> <span class="k">new</span> <span class="n">MyFilter</span><span class="o">())</span>
</code></pre></div>
<p>Suppose the implementation of MyFilter is this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">MyFilter</span> <span class="kd">extends</span> <span class="n">BaseFilter</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">boolean</span> <span class="nf">isKeep</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">boolean</span> <span class="n">isKeep</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="k">return</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getInteger</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> <span class="o"><</span> <span class="mi">10</span><span class="o">;</span>
<span class="o">}</span>
<span class="o">}</span>
@@ -247,26 +247,26 @@
<p>Let's now look at how "function fields" work. Suppose you had this function:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">AddAndMultiply</span> <span class="kd">extends</span> <span class="n">BaseFunction</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="kt">int</span> <span class="n">i1</span> <span class="o">=</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getInteger</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span>
<span class="kt">int</span> <span class="n">i2</span> <span class="o">=</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getInteger</span><span class="o">(</span><span class="mi">1</span><span class="o">);</span>
- <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">i1</span> <span class="o">+</span> <span class="n">i2</span><span class="o">,</span> <span class="n">i1</span> <span class="o">*</span> <span class="n">i2</span><span class="o">));</span>
+ <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">i1</span> <span class="o">+</span> <span class="n">i2</span><span class="o">,</span> <span class="n">i1</span> <span class="o">*</span> <span class="n">i2</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
<p>This function takes two numbers as input and emits two new values: the addition of the numbers and the multiplication of the numbers. Suppose you had a stream with the fields "x", "y", and "z". You would use this function like this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">stream</span><span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"x"</span><span class="o">,</span> <span class="s">"y"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">AddAndMultiply</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"added"</span><span class="o">,</span> <span class="s">"multiplied"</span><span class="o">));</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">stream</span><span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"x"</span><span class="o">,</span> <span class="s">"y"</span><span class="o">),</span> <span class="k">new</span> <span class="n">AddAndMultiply</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"added"</span><span class="o">,</span> <span class="s">"multiplied"</span><span class="o">));</span>
</code></pre></div>
<p>The output of functions is additive: the fields are added to the input tuple. So the output of this each call would contain tuples with the five fields "x", "y", "z", "added", and "multiplied". "added" corresponds to the first value emitted by AddAndMultiply, while "multiplied" corresponds to the second value.</p>
<p>With aggregators, on the other hand, the function fields replace the input tuples. So if you had a stream containing the fields "val1" and "val2", and you did this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">stream</span><span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"val2"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">))</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">stream</span><span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"val2"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">))</span>
</code></pre></div>
<p>The output stream would only contain a single tuple with a single field called "sum", representing the sum of all "val2" fields in that batch.</p>
<p>With grouped streams, the output will contain the grouping fields followed by the fields emitted by the aggregator. For example:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">stream</span><span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"val1"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"val2"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">))</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">stream</span><span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"val1"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"val2"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">))</span>
</code></pre></div>
<p>In this example, the output will contain the fields "val1" and "sum".</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Troubleshooting.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Troubleshooting.html b/_site/documentation/Troubleshooting.html
index 896f59f..e1d392c 100644
--- a/_site/documentation/Troubleshooting.html
+++ b/_site/documentation/Troubleshooting.html
@@ -157,7 +157,7 @@
<li>Try deleting the local dirs for the supervisors and restarting the daemons. Supervisors create a unique id for themselves and store it locally. When that id is copied to other nodes, Storm gets confused. </li>
</ul>
-<h3 id=""multiple-defaults.yaml-found"-error">"Multiple defaults.yaml found" error</h3>
+<h3 id="quot-multiple-defaults-yaml-found-quot-error">"Multiple defaults.yaml found" error</h3>
<p>Symptoms:</p>
@@ -171,7 +171,7 @@
<li>You're most likely including the Storm jars inside your topology jar. When packaging your topology jar, don't include the Storm jars as Storm will put those on the classpath for you.</li>
</ul>
-<h3 id=""nosuchmethoderror"-when-running-storm-jar">"NoSuchMethodError" when running storm jar</h3>
+<h3 id="quot-nosuchmethoderror-quot-when-running-storm-jar">"NoSuchMethodError" when running storm jar</h3>
<p>Symptoms:</p>
@@ -192,7 +192,7 @@
<ul>
<li>At runtime, you get a stack trace like the following:</li>
</ul>
-<div class="highlight"><pre><code class="language-text" data-lang="text">java.lang.RuntimeException: java.util.ConcurrentModificationException
+<div class="highlight"><pre><code class="language-" data-lang="">java.lang.RuntimeException: java.util.ConcurrentModificationException
at backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:84)
at backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:55)
at backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:56)
@@ -233,7 +233,7 @@ Caused by: java.util.ConcurrentModificationException
<ul>
<li>You get a NullPointerException that looks something like:</li>
</ul>
-<div class="highlight"><pre><code class="language-text" data-lang="text">java.lang.RuntimeException: java.lang.NullPointerException
+<div class="highlight"><pre><code class="language-" data-lang="">java.lang.RuntimeException: java.lang.NullPointerException
at backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:84)
at backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:55)
at backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:56)
@@ -252,7 +252,7 @@ Caused by: java.lang.NullPointerException
... 6 more
</code></pre></div>
<p>or </p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">java.lang.RuntimeException: java.lang.NullPointerException
+<div class="highlight"><pre><code class="language-" data-lang="">java.lang.RuntimeException: java.lang.NullPointerException
at
backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:128)
~[storm-core-0.9.3.jar:0.9.3]
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Understanding-the-parallelism-of-a-Storm-topology.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Understanding-the-parallelism-of-a-Storm-topology.html b/_site/documentation/Understanding-the-parallelism-of-a-Storm-topology.html
index 773af59..f2d2b16 100644
--- a/_site/documentation/Understanding-the-parallelism-of-a-Storm-topology.html
+++ b/_site/documentation/Understanding-the-parallelism-of-a-Storm-topology.html
@@ -90,7 +90,7 @@
<p class="post-meta"></p>
-<h2 id="what-makes-a-running-topology:-worker-processes,-executors-and-tasks">What makes a running topology: worker processes, executors and tasks</h2>
+<h2 id="what-makes-a-running-topology-worker-processes-executors-and-tasks">What makes a running topology: worker processes, executors and tasks</h2>
<p>Storm distinguishes between the following three main entities that are used to actually run a topology in a Storm cluster:</p>
@@ -128,7 +128,7 @@
</ul></li>
</ul>
-<h3 id="number-of-executors-(threads)">Number of executors (threads)</h3>
+<h3 id="number-of-executors-threads">Number of executors (threads)</h3>
<ul>
<li>Description: How many executors to spawn <em>per component</em>.</li>
@@ -155,9 +155,9 @@
</ul>
<p>Here is an example code snippet to show these settings in practice:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">topologyBuilder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"green-bolt"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">GreenBolt</span><span class="o">(),</span> <span class="mi">2</span><span class="o">)</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">topologyBuilder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"green-bolt"</span><span class="o">,</span> <span class="k">new</span> <span class="n">GreenBolt</span><span class="o">(),</span> <span class="mi">2</span><span class="o">)</span>
<span class="o">.</span><span class="na">setNumTasks</span><span class="o">(</span><span class="mi">4</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"blue-spout"</span><span class="o">);</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"blue-spout"</span><span class="o">);</span>
</code></pre></div>
<p>In the above code we configured Storm to run the bolt <code>GreenBolt</code> with an initial number of two executors and four associated tasks. Storm will run two tasks per executor (thread). If you do not explicitly configure the number of tasks, Storm will run by default one task per executor.</p>
@@ -168,20 +168,20 @@
<p><img src="images/example-of-a-running-topology.png" alt="Example of a running topology in Storm"></p>
<p>The <code>GreenBolt</code> was configured as per the code snippet above whereas <code>BlueSpout</code> and <code>YellowBolt</code> only set the parallelism hint (number of executors). Here is the relevant code:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Config</span> <span class="n">conf</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Config</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Config</span> <span class="n">conf</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Config</span><span class="o">();</span>
<span class="n">conf</span><span class="o">.</span><span class="na">setNumWorkers</span><span class="o">(</span><span class="mi">2</span><span class="o">);</span> <span class="c1">// use two worker processes</span>
-<span class="n">topologyBuilder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"blue-spout"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">BlueSpout</span><span class="o">(),</span> <span class="mi">2</span><span class="o">);</span> <span class="c1">// set parallelism hint to 2</span>
+<span class="n">topologyBuilder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"blue-spout"</span><span class="o">,</span> <span class="k">new</span> <span class="n">BlueSpout</span><span class="o">(),</span> <span class="mi">2</span><span class="o">);</span> <span class="c1">// set parallelism hint to 2</span>
-<span class="n">topologyBuilder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"green-bolt"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">GreenBolt</span><span class="o">(),</span> <span class="mi">2</span><span class="o">)</span>
+<span class="n">topologyBuilder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"green-bolt"</span><span class="o">,</span> <span class="k">new</span> <span class="n">GreenBolt</span><span class="o">(),</span> <span class="mi">2</span><span class="o">)</span>
<span class="o">.</span><span class="na">setNumTasks</span><span class="o">(</span><span class="mi">4</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"blue-spout"</span><span class="o">);</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"blue-spout"</span><span class="o">);</span>
-<span class="n">topologyBuilder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"yellow-bolt"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">YellowBolt</span><span class="o">(),</span> <span class="mi">6</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"green-bolt"</span><span class="o">);</span>
+<span class="n">topologyBuilder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"yellow-bolt"</span><span class="o">,</span> <span class="k">new</span> <span class="n">YellowBolt</span><span class="o">(),</span> <span class="mi">6</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"green-bolt"</span><span class="o">);</span>
<span class="n">StormSubmitter</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span>
- <span class="s">"mytopology"</span><span class="o">,</span>
+ <span class="s">"mytopology"</span><span class="o">,</span>
<span class="n">conf</span><span class="o">,</span>
<span class="n">topologyBuilder</span><span class="o">.</span><span class="na">createTopology</span><span class="o">()</span>
<span class="o">);</span>
@@ -204,9 +204,9 @@
</ol>
<p>Here is an example of using the CLI tool:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">## Reconfigure the topology "mytopology" to use 5 worker processes,
-## the spout "blue-spout" to use 3 executors and
-## the bolt "yellow-bolt" to use 10 executors.
+<div class="highlight"><pre><code class="language-" data-lang="">## Reconfigure the topology "mytopology" to use 5 worker processes,
+## the spout "blue-spout" to use 3 executors and
+## the bolt "yellow-bolt" to use 10 executors.
$ storm rebalance mytopology -n 5 -e blue-spout=3 -e yellow-bolt=10
</code></pre></div>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Using-non-JVM-languages-with-Storm.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Using-non-JVM-languages-with-Storm.html b/_site/documentation/Using-non-JVM-languages-with-Storm.html
index 1dab5ea..2483509 100644
--- a/_site/documentation/Using-non-JVM-languages-with-Storm.html
+++ b/_site/documentation/Using-non-JVM-languages-with-Storm.html
@@ -127,7 +127,7 @@
<p>The right place to start is src/storm.thrift. Since Storm topologies are just Thrift structures, and Nimbus is a Thrift daemon, you can create and submit topologies in any language.</p>
<p>When you create the Thrift structs for spouts and bolts, the code for the spout or bolt is specified in the ComponentObject struct:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">union ComponentObject {
+<div class="highlight"><pre><code class="language-" data-lang="">union ComponentObject {
1: binary serialized_java;
2: ShellComponent shell;
3: JavaObject java_object;
@@ -136,13 +136,13 @@
<p>For a non-JVM DSL, you would want to make use of "2" and "3". ShellComponent lets you specify a script to run that component (e.g., your python code). And JavaObject lets you specify native java spouts and bolts for the component (and Storm will use reflection to create that spout or bolt).</p>
<p>There's a "storm shell" command that will help with submitting a topology. Its usage is like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">storm shell resources/ python topology.py arg1 arg2
+<div class="highlight"><pre><code class="language-" data-lang="">storm shell resources/ python topology.py arg1 arg2
</code></pre></div>
<p>storm shell will then package resources/ into a jar, upload the jar to Nimbus, and call your topology.py script like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">python topology.py arg1 arg2 {nimbus-host} {nimbus-port} {uploaded-jar-location}
+<div class="highlight"><pre><code class="language-" data-lang="">python topology.py arg1 arg2 {nimbus-host} {nimbus-port} {uploaded-jar-location}
</code></pre></div>
<p>Then you can connect to Nimbus using the Thrift API and submit the topology, passing {uploaded-jar-location} into the submitTopology method. For reference, here's the submitTopology definition:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">void submitTopology(1: string name, 2: string uploadedJarLocation, 3: string jsonConf, 4: StormTopology topology)
+<div class="highlight"><pre><code class="language-" data-lang="">void submitTopology(1: string name, 2: string uploadedJarLocation, 3: string jsonConf, 4: StormTopology topology)
throws (1: AlreadyAliveException e, 2: InvalidTopologyException ite);
</code></pre></div>
[09/10] storm git commit: STORM-1366. Add documentation for StormSQL
into the website.
Posted by wh...@apache.org.
STORM-1366. Add documentation for StormSQL into the website.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/dfa66d7d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/dfa66d7d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/dfa66d7d
Branch: refs/heads/asf-site
Commit: dfa66d7d5ef593fa94b4ec7c1900f3a08083be48
Parents: 2ba4f17
Author: Haohui Mai <wh...@apache.org>
Authored: Fri Jan 8 15:48:42 2016 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Sat Jan 9 17:20:07 2016 -0800
----------------------------------------------------------------------
_site/2012/08/02/storm080-released.html | 2 +-
_site/2012/09/06/storm081-released.html | 2 +-
_site/2013/12/08/storm090-released.html | 4 +-
_site/2014/06/25/storm092-released.html | 2 +-
_site/2015/06/15/storm0100-beta-released.html | 4 +-
_site/contribute/BYLAWS.html | 8 +-
_site/contribute/Contributing-to-Storm.html | 1 +
_site/doc-index.html | 8 +
_site/documentation.html | 5 +-
.../Acking-framework-implementation.html | 2 +-
_site/documentation/Clojure-DSL.html | 182 +++----
_site/documentation/Common-patterns.html | 36 +-
.../Creating-a-new-Storm-project.html | 2 +-
...fining-a-non-jvm-language-dsl-for-storm.html | 8 +-
_site/documentation/Distributed-RPC.html | 62 +--
_site/documentation/FAQ.html | 30 +-
_site/documentation/Fault-tolerance.html | 10 +-
.../Guaranteeing-message-processing.html | 64 +--
_site/documentation/Home.html | 4 +-
.../Installing-native-dependencies.html | 4 +-
_site/documentation/Kestrel-and-Storm.html | 65 +--
_site/documentation/Local-mode.html | 2 +-
_site/documentation/Multilang-protocol.html | 236 ++++-----
...ning-topologies-on-a-production-cluster.html | 4 +-
.../Serialization-(prior-to-0.6.0).html | 6 +-
_site/documentation/Serialization.html | 8 +-
.../Setting-up-a-Storm-cluster.html | 26 +-
.../Setting-up-development-environment.html | 6 +-
...age-protocol-(versions-0.7.0-and-below).html | 86 ++--
.../Structure-of-the-codebase.html | 4 +-
.../documentation/Transactional-topologies.html | 44 +-
_site/documentation/Trident-API-Overview.html | 84 +--
_site/documentation/Trident-spouts.html | 4 +-
_site/documentation/Trident-state.html | 130 ++---
_site/documentation/Trident-tutorial.html | 84 +--
_site/documentation/Troubleshooting.html | 10 +-
...ing-the-parallelism-of-a-Storm-topology.html | 28 +-
.../Using-non-JVM-languages-with-Storm.html | 8 +-
_site/documentation/flux.html | 510 ++++++++++---------
_site/documentation/getting-help.html | 4 +-
_site/documentation/storm-eventhubs.html | 8 +-
_site/documentation/storm-hbase.html | 94 ++--
_site/documentation/storm-hdfs.html | 119 ++---
_site/documentation/storm-hive.html | 42 +-
_site/documentation/storm-jdbc.html | 112 ++--
_site/documentation/storm-kafka.html | 93 ++--
_site/documentation/storm-redis.html | 106 ++--
_site/documentation/storm-solr.html | 32 +-
_site/feed.xml | 32 +-
_site/getting-help.html | 4 +-
_site/tutorial.html | 92 ++--
documentation.html | 5 +-
.../images/storm-sql-internal-example.png | Bin 0 -> 28377 bytes
.../images/storm-sql-internal-workflow.png | Bin 0 -> 20020 bytes
documentation/storm-sql-internal.md | 55 ++
documentation/storm-sql.md | 97 ++++
56 files changed, 1432 insertions(+), 1248 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/2012/08/02/storm080-released.html
----------------------------------------------------------------------
diff --git a/_site/2012/08/02/storm080-released.html b/_site/2012/08/02/storm080-released.html
index d90f9ed..70d0102 100644
--- a/_site/2012/08/02/storm080-released.html
+++ b/_site/2012/08/02/storm080-released.html
@@ -202,7 +202,7 @@ topology.transfer.buffer.size </p>
<p>These may require some tweaking to optimize your topologies, but most likely the default values will work fine for you out of the box. </p>
-<h2 id="decreased-zookeeper-load-/-increased-storm-ui-performance">Decreased Zookeeper load / increased Storm UI performance</h2>
+<h2 id="decreased-zookeeper-load-increased-storm-ui-performance">Decreased Zookeeper load / increased Storm UI performance</h2>
<p>Storm sends significantly less traffic to Zookeeper now (on the order of 10x less). Since it also uses so many fewer znodes to store state, the UI is significantly faster as well. </p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/2012/09/06/storm081-released.html
----------------------------------------------------------------------
diff --git a/_site/2012/09/06/storm081-released.html b/_site/2012/09/06/storm081-released.html
index f3b515e..c1c8fa8 100644
--- a/_site/2012/09/06/storm081-released.html
+++ b/_site/2012/09/06/storm081-released.html
@@ -154,7 +154,7 @@
<div>
<p>Storm 0.8.1 is now available on the downloads page and in Maven. This release contains many bug fixes as well as a few important new features. These include: </p>
-<h2 id="storm's-unit-testing-facilities-have-been-exposed-via-java">Storm's unit testing facilities have been exposed via Java</h2>
+<h2 id="storm-39-s-unit-testing-facilities-have-been-exposed-via-java">Storm's unit testing facilities have been exposed via Java</h2>
<p>This is an extremely powerful API that lets you do things like:
a) Easily bring up and tear down local clusters
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/2013/12/08/storm090-released.html
----------------------------------------------------------------------
diff --git a/_site/2013/12/08/storm090-released.html b/_site/2013/12/08/storm090-released.html
index 15670f2..c4f3a30 100644
--- a/_site/2013/12/08/storm090-released.html
+++ b/_site/2013/12/08/storm090-released.html
@@ -165,7 +165,7 @@
<p>The Netty transport offers a pure Java alternative that eliminates Storm's dependency on native libraries. The Netty transport's performance is up to twice as fast as 0MQ, and it will open the door for authorization and authentication between worker processes. For an in-depth performance comparison of the 0MQ and Netty transports, see <a href="http://yahooeng.tumblr.com/post/64758709722/making-storm-fly-with-netty">this blog post</a> by Storm contributor <a href="https://github.com/revans2">Bobby Evans</a>.</p>
<p>To configure Storm to use the Netty transport simply add the following to your <code>storm.yaml</code> configuration and adjust the values to best suit your use-case:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">storm.messaging.transport: "backtype.storm.messaging.netty.Context"
+<div class="highlight"><pre><code class="language-" data-lang="">storm.messaging.transport: "backtype.storm.messaging.netty.Context"
storm.messaging.netty.server_worker_threads: 1
storm.messaging.netty.client_worker_threads: 1
storm.messaging.netty.buffer_size: 5242880
@@ -182,7 +182,7 @@ storm.messaging.netty.min_wait_ms: 100
<p>In earlier versions of Storm, viewing worker logs involved determining a worker's location (host/port), typically through Storm UI, then <code>ssh</code>ing to that host and <code>tail</code>ing the corresponding worker log file. With the new log viewer. You can now easily access a specific worker's log in a web browser by clicking on a worker's port number right from Storm UI.</p>
<p>The <code>logviewer</code> daemon runs as a separate process on Storm supervisor nodes. To enable the <code>logviewer</code> run the following command (under supervision) on your cluster's supervisor nodes:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">$ storm logviewer
+<div class="highlight"><pre><code class="language-" data-lang="">$ storm logviewer
</code></pre></div>
<h2 id="improved-windows-support">Improved Windows Support</h2>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/2014/06/25/storm092-released.html
----------------------------------------------------------------------
diff --git a/_site/2014/06/25/storm092-released.html b/_site/2014/06/25/storm092-released.html
index ad5d276..4ea77b6 100644
--- a/_site/2014/06/25/storm092-released.html
+++ b/_site/2014/06/25/storm092-released.html
@@ -187,7 +187,7 @@
<p>The <code>storm-kafka</code> module can be found in the <code>/external/</code> directory of the source tree and binary distributions. The <code>external</code> area has been set up to contain projects that while not required by Storm, are often used in conjunction with Storm to integrate with some other technology. Such projects also come with a maintenance committment from at least one Storm committer to ensure compatibility with Storm's main codebase as it evolves.</p>
<p>The <code>storm-kafka</code> dependency is available now from Maven Central at the following coordinates:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">groupId: org.apache.storm
+<div class="highlight"><pre><code class="language-" data-lang="">groupId: org.apache.storm
artifactId: storm-kafka
version: 0.9.2-incubating
</code></pre></div>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/2015/06/15/storm0100-beta-released.html
----------------------------------------------------------------------
diff --git a/_site/2015/06/15/storm0100-beta-released.html b/_site/2015/06/15/storm0100-beta-released.html
index f9f3341..03d7dcf 100644
--- a/_site/2015/06/15/storm0100-beta-released.html
+++ b/_site/2015/06/15/storm0100-beta-released.html
@@ -156,7 +156,7 @@
<p>Aside from many stability and performance improvements, this release includes a number of important new features, some of which are highlighted below.</p>
-<h2 id="secure,-multi-tenant-deployment">Secure, Multi-Tenant Deployment</h2>
+<h2 id="secure-multi-tenant-deployment">Secure, Multi-Tenant Deployment</h2>
<p>Much like the early days of Hadoop, Apache Storm originally evolved in an environment where security was not a high-priority concern. Rather, it was assumed that Storm would be deployed to environments suitably cordoned off from security threats. While a large number of users were comfortable setting up their own security measures for Storm (usually at the Firewall/OS level), this proved a hindrance to broader adoption among larger enterprises where security policies prohibited deployment without specific safeguards.</p>
@@ -244,7 +244,7 @@ that allows users to stream data from Storm directly into hive. Storm's Hive
<p>Further information can be found in the <a href="https://github.com/apache/storm/blob/v0.10.0-beta/external/storm-redis/README.md">storm-redis documentation</a>.</p>
-<h2 id="jdbc/rdbms-integration">JDBC/RDBMS Integration</h2>
+<h2 id="jdbc-rdbms-integration">JDBC/RDBMS Integration</h2>
<p>Many stream processing data flows require accessing data from or writing data to a relational data store. Storm 0.10.0 introduces highly flexible and customizable support for integrating with virtually any JDBC-compliant database.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/contribute/BYLAWS.html
----------------------------------------------------------------------
diff --git a/_site/contribute/BYLAWS.html b/_site/contribute/BYLAWS.html
index 007bcbb..ffe2f75 100644
--- a/_site/contribute/BYLAWS.html
+++ b/_site/contribute/BYLAWS.html
@@ -94,17 +94,17 @@
<p>Apache projects define a set of roles with associated rights and responsibilities. These roles govern what tasks an individual may perform within the project. The roles are defined in the following sections:</p>
-<h3 id="users:">Users:</h3>
+<h3 id="users">Users:</h3>
<p>The most important participants in the project are people who use our software. The majority of our developers start out as users and guide their development efforts from the user's perspective.</p>
<p>Users contribute to the Apache projects by providing feedback to developers in the form of bug reports and feature suggestions. As well, users participate in the Apache community by helping other users on mailing lists and user support forums.</p>
-<h3 id="contributors:">Contributors:</h3>
+<h3 id="contributors">Contributors:</h3>
<p>Contributors are all of the volunteers who are contributing time, code, documentation, or resources to the Storm Project. A contributor that makes sustained, welcome contributions to the project may be invited to become a Committer, though the exact timing of such invitations depends on many factors.</p>
-<h3 id="committers:">Committers:</h3>
+<h3 id="committers">Committers:</h3>
<p>The project's Committers are responsible for the project's technical management. Committers have access to all project source repositories. Committers may cast binding votes on any technical discussion regarding storm.</p>
@@ -114,7 +114,7 @@
<p>A Committer who makes a sustained contribution to the project may be invited to become a member of the PMC. The form of contribution is not limited to code. It can also include code review, helping out users on the mailing lists, documentation, testing, etc.</p>
-<h3 id="project-management-committee(pmc):">Project Management Committee(PMC):</h3>
+<h3 id="project-management-committee-pmc">Project Management Committee(PMC):</h3>
<p>The PMC is responsible to the board and the ASF for the management and oversight of the Apache Storm codebase. The responsibilities of the PMC include:</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/contribute/Contributing-to-Storm.html
----------------------------------------------------------------------
diff --git a/_site/contribute/Contributing-to-Storm.html b/_site/contribute/Contributing-to-Storm.html
index b174582..217e646 100644
--- a/_site/contribute/Contributing-to-Storm.html
+++ b/_site/contribute/Contributing-to-Storm.html
@@ -108,6 +108,7 @@
<li>Open an issue on the <a href="https://issues.apache.org/jira/browse/STORM">JIRA issue tracker</a> if one doesn't exist already</li>
<li>Comment on the issue with your plan for implementing the issue. Explain what pieces of the codebase you're going to touch and how everything is going to fit together.</li>
<li>Storm committers will iterate with you on the design to make sure you're on the right track</li>
+<li>Read through the developer documentation on how to build, code style, testing, etc <a href="https://github.com/apache/storm/blob/master/DEVELOPER.md">DEVELOPER.md</a> </li>
<li>Implement your issue, submit a pull request prefixed with the JIRA ID (e.g. "STORM-123: add new feature foo"), and iterate from there.</li>
</ol>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/doc-index.html
----------------------------------------------------------------------
diff --git a/_site/doc-index.html b/_site/doc-index.html
index 6195898..0525227 100644
--- a/_site/doc-index.html
+++ b/_site/doc-index.html
@@ -341,6 +341,14 @@
<!-- resource-p -->
+<li><a href="/documentation/storm-sql-internal.html">The Internals of Storm SQL</a></li>
+ <!-- resource-p -->
+
+
+<li><a href="/documentation/storm-sql.html">Storm SQL integration</a></li>
+ <!-- resource-p -->
+
+
<li><a href="/talksAndVideos.html">Resources</a></li>
<!-- resource-p -->
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation.html
----------------------------------------------------------------------
diff --git a/_site/documentation.html b/_site/documentation.html
index aba3871..ddb54bb 100644
--- a/_site/documentation.html
+++ b/_site/documentation.html
@@ -133,8 +133,8 @@
</ul>
</div>
<div role="tabpanel" class="tab-pane" id="integration">
- <p>The following modules are included in the Apache Storm distribution and are not required for storm to operate,
- but are useful for extending Storm in order to provide additional functionality such as integration with other
+ <p>The following modules are included in the Apache Storm distribution and are not required for storm to operate,
+ but are useful for extending Storm in order to provide additional functionality such as integration with other
technologies frequently used in combination with Storm.</p>
<ul>
<li><a href="documentation/storm-kafka.html">Kafka</a></li>
@@ -146,6 +146,7 @@
<li><a href="documentation/storm-solr.html">Solr</a></li>
<li><a href="documentation/storm-eventhubs.html">Azure EventHubs</a></li>
<li><a href="documentation/flux.html">Flux</a> (declarative wiring/configuration of Topologies)</li>
+ <li><a href="documentation/storm-sql.html">SQL</a> (writing topologies in SQL)</li>
</ul>
</div>
<div role="tabpanel" class="tab-pane" id="intermediate">
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Acking-framework-implementation.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Acking-framework-implementation.html b/_site/documentation/Acking-framework-implementation.html
index a704d51..6de92ff 100644
--- a/_site/documentation/Acking-framework-implementation.html
+++ b/_site/documentation/Acking-framework-implementation.html
@@ -94,7 +94,7 @@
<p>You can read a bit more about the <a href="Guaranteeing-message-processing.html#what-is-storms-reliability-api">reliability mechanism</a> elsewhere on the wiki -- this explains the internal details.</p>
-<h3 id="acker-execute()">acker <code>execute()</code></h3>
+<h3 id="acker-execute">acker <code>execute()</code></h3>
<p>The acker is actually a regular bolt, with its <a href="https://github.com/apache/storm/blob/46c3ba7/storm-core/src/clj/backtype/storm/daemon/acker.clj#L36">execute method</a> defined withing <code>mk-acker-bolt</code>. When a new tupletree is born, the spout sends the XORed edge-ids of each tuple recipient, which the acker records in its <code>pending</code> ledger. Every time an executor acks a tuple, the acker receives a partial checksum that is the XOR of the tuple's own edge-id (clearing it from the ledger) and the edge-id of each downstream tuple the executor emitted (thus entering them into the ledger).</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Clojure-DSL.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Clojure-DSL.html b/_site/documentation/Clojure-DSL.html
index bcb19f4..d7ba55e 100644
--- a/_site/documentation/Clojure-DSL.html
+++ b/_site/documentation/Clojure-DSL.html
@@ -107,19 +107,19 @@
<p>To define a topology, use the <code>topology</code> function. <code>topology</code> takes in two arguments: a map of "spout specs" and a map of "bolt specs". Each spout and bolt spec wires the code for the component into the topology by specifying things like inputs and parallelism.</p>
<p>Let's take a look at an example topology definition <a href="https://github.com/apache/storm/blob/master/examples/storm-starter/src/clj/storm/starter/clj/word_count.clj">from the storm-starter project</a>:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">topology</span>
- <span class="p">{</span><span class="s">"1"</span> <span class="p">(</span><span class="nf">spout-spec</span> <span class="nv">sentence-spout</span><span class="p">)</span>
- <span class="s">"2"</span> <span class="p">(</span><span class="nf">spout-spec</span> <span class="p">(</span><span class="nf">sentence-spout-parameterized</span>
- <span class="p">[</span><span class="s">"the cat jumped over the door"</span>
- <span class="s">"greetings from a faraway land"</span><span class="p">])</span>
- <span class="ss">:p</span> <span class="mi">2</span><span class="p">)}</span>
- <span class="p">{</span><span class="s">"3"</span> <span class="p">(</span><span class="nf">bolt-spec</span> <span class="p">{</span><span class="s">"1"</span> <span class="ss">:shuffle</span> <span class="s">"2"</span> <span class="ss">:shuffle</span><span class="p">}</span>
- <span class="nv">split-sentence</span>
- <span class="ss">:p</span> <span class="mi">5</span><span class="p">)</span>
- <span class="s">"4"</span> <span class="p">(</span><span class="nf">bolt-spec</span> <span class="p">{</span><span class="s">"3"</span> <span class="p">[</span><span class="s">"word"</span><span class="p">]}</span>
- <span class="nv">word-count</span>
- <span class="ss">:p</span> <span class="mi">6</span><span class="p">)})</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">topology</span><span class="w">
+ </span><span class="p">{</span><span class="s">"1"</span><span class="w"> </span><span class="p">(</span><span class="nf">spout-spec</span><span class="w"> </span><span class="n">sentence-spout</span><span class="p">)</span><span class="w">
+ </span><span class="s">"2"</span><span class="w"> </span><span class="p">(</span><span class="nf">spout-spec</span><span class="w"> </span><span class="p">(</span><span class="nf">sentence-spout-parameterized</span><span class="w">
+ </span><span class="p">[</span><span class="s">"the cat jumped over the door"</span><span class="w">
+ </span><span class="s">"greetings from a faraway land"</span><span class="p">])</span><span class="w">
+ </span><span class="no">:p</span><span class="w"> </span><span class="mi">2</span><span class="p">)}</span><span class="w">
+ </span><span class="p">{</span><span class="s">"3"</span><span class="w"> </span><span class="p">(</span><span class="nf">bolt-spec</span><span class="w"> </span><span class="p">{</span><span class="s">"1"</span><span class="w"> </span><span class="no">:shuffle</span><span class="w"> </span><span class="s">"2"</span><span class="w"> </span><span class="no">:shuffle</span><span class="p">}</span><span class="w">
+ </span><span class="n">split-sentence</span><span class="w">
+ </span><span class="no">:p</span><span class="w"> </span><span class="mi">5</span><span class="p">)</span><span class="w">
+ </span><span class="s">"4"</span><span class="w"> </span><span class="p">(</span><span class="nf">bolt-spec</span><span class="w"> </span><span class="p">{</span><span class="s">"3"</span><span class="w"> </span><span class="p">[</span><span class="s">"word"</span><span class="p">]}</span><span class="w">
+ </span><span class="n">word-count</span><span class="w">
+ </span><span class="no">:p</span><span class="w"> </span><span class="mi">6</span><span class="p">)})</span><span class="w">
+</span></code></pre></div>
<p>The maps of spout and bolt specs are maps from the component id to the corresponding spec. The component ids must be unique across the maps. Just like defining topologies in Java, component ids are used when declaring inputs for bolts in the topology.</p>
<h4 id="spout-spec">spout-spec</h4>
@@ -148,10 +148,10 @@
</ol>
<p>See <a href="Concepts.html">Concepts</a> for more info on stream groupings. Here's an example input declaration showcasing the various ways to declare inputs:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">{[</span><span class="s">"2"</span> <span class="s">"1"</span><span class="p">]</span> <span class="ss">:shuffle</span>
- <span class="s">"3"</span> <span class="p">[</span><span class="s">"field1"</span> <span class="s">"field2"</span><span class="p">]</span>
- <span class="p">[</span><span class="s">"4"</span> <span class="s">"2"</span><span class="p">]</span> <span class="ss">:global</span><span class="p">}</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">{[</span><span class="s">"2"</span><span class="w"> </span><span class="s">"1"</span><span class="p">]</span><span class="w"> </span><span class="no">:shuffle</span><span class="w">
+ </span><span class="s">"3"</span><span class="w"> </span><span class="p">[</span><span class="s">"field1"</span><span class="w"> </span><span class="s">"field2"</span><span class="p">]</span><span class="w">
+ </span><span class="p">[</span><span class="s">"4"</span><span class="w"> </span><span class="s">"2"</span><span class="p">]</span><span class="w"> </span><span class="no">:global</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>This input declaration subscribes to three streams total. It subscribes to stream "1" on component "2" with a shuffle grouping, subscribes to the default stream on component "3" with a fields grouping on the fields "field1" and "field2", and subscribes to stream "2" on component "4" with a global grouping.</p>
<p>Like <code>spout-spec</code>, the only current supported keyword argument for <code>bolt-spec</code> is <code>:p</code> which specifies the parallelism for the bolt.</p>
@@ -161,12 +161,12 @@
<p><code>shell-bolt-spec</code> is used for defining bolts that are implemented in a non-JVM language. It takes as arguments the input declaration, the command line program to run, the name of the file implementing the bolt, an output specification, and then the same keyword arguments that <code>bolt-spec</code> accepts.</p>
<p>Here's an example <code>shell-bolt-spec</code>:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">shell-bolt-spec</span> <span class="p">{</span><span class="s">"1"</span> <span class="ss">:shuffle</span> <span class="s">"2"</span> <span class="p">[</span><span class="s">"id"</span><span class="p">]}</span>
- <span class="s">"python"</span>
- <span class="s">"mybolt.py"</span>
- <span class="p">[</span><span class="s">"outfield1"</span> <span class="s">"outfield2"</span><span class="p">]</span>
- <span class="ss">:p</span> <span class="mi">25</span><span class="p">)</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">shell-bolt-spec</span><span class="w"> </span><span class="p">{</span><span class="s">"1"</span><span class="w"> </span><span class="no">:shuffle</span><span class="w"> </span><span class="s">"2"</span><span class="w"> </span><span class="p">[</span><span class="s">"id"</span><span class="p">]}</span><span class="w">
+ </span><span class="s">"python"</span><span class="w">
+ </span><span class="s">"mybolt.py"</span><span class="w">
+ </span><span class="p">[</span><span class="s">"outfield1"</span><span class="w"> </span><span class="s">"outfield2"</span><span class="p">]</span><span class="w">
+ </span><span class="no">:p</span><span class="w"> </span><span class="mi">25</span><span class="p">)</span><span class="w">
+</span></code></pre></div>
<p>The syntax of output declarations is described in more detail in the <code>defbolt</code> section below. See <a href="Using-non-JVM-languages-with-Storm.html">Using non JVM languages with Storm</a> for more details on how multilang works within Storm.</p>
<h3 id="defbolt">defbolt</h3>
@@ -182,47 +182,47 @@
<h4 id="simple-bolts">Simple bolts</h4>
<p>Let's start with the simplest form of <code>defbolt</code>. Here's an example bolt that splits a tuple containing a sentence into a tuple for each word:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defbolt</span> <span class="nv">split-sentence</span> <span class="p">[</span><span class="s">"word"</span><span class="p">]</span> <span class="p">[</span><span class="nv">tuple</span> <span class="nv">collector</span><span class="p">]</span>
- <span class="p">(</span><span class="k">let </span><span class="p">[</span><span class="nv">words</span> <span class="p">(</span><span class="nf">.split</span> <span class="p">(</span><span class="nf">.getString</span> <span class="nv">tuple</span> <span class="mi">0</span><span class="p">)</span> <span class="s">" "</span><span class="p">)]</span>
- <span class="p">(</span><span class="nb">doseq </span><span class="p">[</span><span class="nv">w</span> <span class="nv">words</span><span class="p">]</span>
- <span class="p">(</span><span class="nf">emit-bolt!</span> <span class="nv">collector</span> <span class="p">[</span><span class="nv">w</span><span class="p">]</span> <span class="ss">:anchor</span> <span class="nv">tuple</span><span class="p">))</span>
- <span class="p">(</span><span class="nf">ack!</span> <span class="nv">collector</span> <span class="nv">tuple</span><span class="p">)</span>
- <span class="p">))</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defbolt</span><span class="w"> </span><span class="n">split-sentence</span><span class="w"> </span><span class="p">[</span><span class="s">"word"</span><span class="p">]</span><span class="w"> </span><span class="p">[</span><span class="n">tuple</span><span class="w"> </span><span class="n">collector</span><span class="p">]</span><span class="w">
+ </span><span class="p">(</span><span class="k">let</span><span class="w"> </span><span class="p">[</span><span class="n">words</span><span class="w"> </span><span class="p">(</span><span class="nf">.split</span><span class="w"> </span><span class="p">(</span><span class="nf">.getString</span><span class="w"> </span><span class="n">tuple</span><span class="w"> </span><span class="mi">0</span><span class="p">)</span><span class="w"> </span><span class="s">" "</span><span class="p">)]</span><span class="w">
+ </span><span class="p">(</span><span class="nb">doseq</span><span class="w"> </span><span class="p">[</span><span class="n">w</span><span class="w"> </span><span class="n">words</span><span class="p">]</span><span class="w">
+ </span><span class="p">(</span><span class="nf">emit-bolt!</span><span class="w"> </span><span class="n">collector</span><span class="w"> </span><span class="p">[</span><span class="n">w</span><span class="p">]</span><span class="w"> </span><span class="no">:anchor</span><span class="w"> </span><span class="n">tuple</span><span class="p">))</span><span class="w">
+ </span><span class="p">(</span><span class="nf">ack!</span><span class="w"> </span><span class="n">collector</span><span class="w"> </span><span class="n">tuple</span><span class="p">)</span><span class="w">
+ </span><span class="p">))</span><span class="w">
+</span></code></pre></div>
<p>Since the option map is omitted, this is a non-prepared bolt. The DSL simply expects an implementation for the <code>execute</code> method of <code>IRichBolt</code>. The implementation takes two parameters, the tuple and the <code>OutputCollector</code>, and is followed by the body of the <code>execute</code> function. The DSL automatically type-hints the parameters for you so you don't need to worry about reflection if you use Java interop.</p>
<p>This implementation binds <code>split-sentence</code> to an actual <code>IRichBolt</code> object that you can use in topologies, like so:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">bolt-spec</span> <span class="p">{</span><span class="s">"1"</span> <span class="ss">:shuffle</span><span class="p">}</span>
- <span class="nv">split-sentence</span>
- <span class="ss">:p</span> <span class="mi">5</span><span class="p">)</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">bolt-spec</span><span class="w"> </span><span class="p">{</span><span class="s">"1"</span><span class="w"> </span><span class="no">:shuffle</span><span class="p">}</span><span class="w">
+ </span><span class="n">split-sentence</span><span class="w">
+ </span><span class="no">:p</span><span class="w"> </span><span class="mi">5</span><span class="p">)</span><span class="w">
+</span></code></pre></div>
<h4 id="parameterized-bolts">Parameterized bolts</h4>
<p>Many times you want to parameterize your bolts with other arguments. For example, let's say you wanted to have a bolt that appends a suffix to every input string it receives, and you want that suffix to be set at runtime. You do this with <code>defbolt</code> by including a <code>:params</code> option in the option map, like so:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defbolt</span> <span class="nv">suffix-appender</span> <span class="p">[</span><span class="s">"word"</span><span class="p">]</span> <span class="p">{</span><span class="ss">:params</span> <span class="p">[</span><span class="nv">suffix</span><span class="p">]}</span>
- <span class="p">[</span><span class="nv">tuple</span> <span class="nv">collector</span><span class="p">]</span>
- <span class="p">(</span><span class="nf">emit-bolt!</span> <span class="nv">collector</span> <span class="p">[(</span><span class="nb">str </span><span class="p">(</span><span class="nf">.getString</span> <span class="nv">tuple</span> <span class="mi">0</span><span class="p">)</span> <span class="nv">suffix</span><span class="p">)]</span> <span class="ss">:anchor</span> <span class="nv">tuple</span><span class="p">)</span>
- <span class="p">)</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defbolt</span><span class="w"> </span><span class="n">suffix-appender</span><span class="w"> </span><span class="p">[</span><span class="s">"word"</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="no">:params</span><span class="w"> </span><span class="p">[</span><span class="n">suffix</span><span class="p">]}</span><span class="w">
+ </span><span class="p">[</span><span class="n">tuple</span><span class="w"> </span><span class="n">collector</span><span class="p">]</span><span class="w">
+ </span><span class="p">(</span><span class="nf">emit-bolt!</span><span class="w"> </span><span class="n">collector</span><span class="w"> </span><span class="p">[(</span><span class="nb">str</span><span class="w"> </span><span class="p">(</span><span class="nf">.getString</span><span class="w"> </span><span class="n">tuple</span><span class="w"> </span><span class="mi">0</span><span class="p">)</span><span class="w"> </span><span class="n">suffix</span><span class="p">)]</span><span class="w"> </span><span class="no">:anchor</span><span class="w"> </span><span class="n">tuple</span><span class="p">)</span><span class="w">
+ </span><span class="p">)</span><span class="w">
+</span></code></pre></div>
<p>Unlike the previous example, <code>suffix-appender</code> will be bound to a function that returns an <code>IRichBolt</code> rather than be an <code>IRichBolt</code> object directly. This is caused by specifying <code>:params</code> in its option map. So to use <code>suffix-appender</code> in a topology, you would do something like:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">bolt-spec</span> <span class="p">{</span><span class="s">"1"</span> <span class="ss">:shuffle</span><span class="p">}</span>
- <span class="p">(</span><span class="nf">suffix-appender</span> <span class="s">"-suffix"</span><span class="p">)</span>
- <span class="ss">:p</span> <span class="mi">10</span><span class="p">)</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">bolt-spec</span><span class="w"> </span><span class="p">{</span><span class="s">"1"</span><span class="w"> </span><span class="no">:shuffle</span><span class="p">}</span><span class="w">
+ </span><span class="p">(</span><span class="nf">suffix-appender</span><span class="w"> </span><span class="s">"-suffix"</span><span class="p">)</span><span class="w">
+ </span><span class="no">:p</span><span class="w"> </span><span class="mi">10</span><span class="p">)</span><span class="w">
+</span></code></pre></div>
<h4 id="prepared-bolts">Prepared bolts</h4>
<p>To do more complex bolts, such as ones that do joins and streaming aggregations, the bolt needs to store state. You can do this by creating a prepared bolt which is specified by including <code>{:prepare true}</code> in the option map. Consider, for example, this bolt that implements word counting:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defbolt</span> <span class="nv">word-count</span> <span class="p">[</span><span class="s">"word"</span> <span class="s">"count"</span><span class="p">]</span> <span class="p">{</span><span class="ss">:prepare</span> <span class="nv">true</span><span class="p">}</span>
- <span class="p">[</span><span class="nv">conf</span> <span class="nv">context</span> <span class="nv">collector</span><span class="p">]</span>
- <span class="p">(</span><span class="k">let </span><span class="p">[</span><span class="nv">counts</span> <span class="p">(</span><span class="nf">atom</span> <span class="p">{})]</span>
- <span class="p">(</span><span class="nf">bolt</span>
- <span class="p">(</span><span class="nf">execute</span> <span class="p">[</span><span class="nv">tuple</span><span class="p">]</span>
- <span class="p">(</span><span class="k">let </span><span class="p">[</span><span class="nv">word</span> <span class="p">(</span><span class="nf">.getString</span> <span class="nv">tuple</span> <span class="mi">0</span><span class="p">)]</span>
- <span class="p">(</span><span class="nf">swap!</span> <span class="nv">counts</span> <span class="p">(</span><span class="nb">partial merge-with </span><span class="nv">+</span><span class="p">)</span> <span class="p">{</span><span class="nv">word</span> <span class="mi">1</span><span class="p">})</span>
- <span class="p">(</span><span class="nf">emit-bolt!</span> <span class="nv">collector</span> <span class="p">[</span><span class="nv">word</span> <span class="p">(</span><span class="o">@</span><span class="nv">counts</span> <span class="nv">word</span><span class="p">)]</span> <span class="ss">:anchor</span> <span class="nv">tuple</span><span class="p">)</span>
- <span class="p">(</span><span class="nf">ack!</span> <span class="nv">collector</span> <span class="nv">tuple</span><span class="p">)</span>
- <span class="p">)))))</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defbolt</span><span class="w"> </span><span class="n">word-count</span><span class="w"> </span><span class="p">[</span><span class="s">"word"</span><span class="w"> </span><span class="s">"count"</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="no">:prepare</span><span class="w"> </span><span class="n">true</span><span class="p">}</span><span class="w">
+ </span><span class="p">[</span><span class="n">conf</span><span class="w"> </span><span class="n">context</span><span class="w"> </span><span class="n">collector</span><span class="p">]</span><span class="w">
+ </span><span class="p">(</span><span class="k">let</span><span class="w"> </span><span class="p">[</span><span class="n">counts</span><span class="w"> </span><span class="p">(</span><span class="nf">atom</span><span class="w"> </span><span class="p">{})]</span><span class="w">
+ </span><span class="p">(</span><span class="nf">bolt</span><span class="w">
+ </span><span class="p">(</span><span class="nf">execute</span><span class="w"> </span><span class="p">[</span><span class="n">tuple</span><span class="p">]</span><span class="w">
+ </span><span class="p">(</span><span class="k">let</span><span class="w"> </span><span class="p">[</span><span class="n">word</span><span class="w"> </span><span class="p">(</span><span class="nf">.getString</span><span class="w"> </span><span class="n">tuple</span><span class="w"> </span><span class="mi">0</span><span class="p">)]</span><span class="w">
+ </span><span class="p">(</span><span class="nf">swap!</span><span class="w"> </span><span class="n">counts</span><span class="w"> </span><span class="p">(</span><span class="nb">partial</span><span class="w"> </span><span class="nb">merge-with</span><span class="w"> </span><span class="nb">+</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="n">word</span><span class="w"> </span><span class="mi">1</span><span class="p">})</span><span class="w">
+ </span><span class="p">(</span><span class="nf">emit-bolt!</span><span class="w"> </span><span class="n">collector</span><span class="w"> </span><span class="p">[</span><span class="n">word</span><span class="w"> </span><span class="p">(</span><span class="err">@</span><span class="n">counts</span><span class="w"> </span><span class="n">word</span><span class="p">)]</span><span class="w"> </span><span class="no">:anchor</span><span class="w"> </span><span class="n">tuple</span><span class="p">)</span><span class="w">
+ </span><span class="p">(</span><span class="nf">ack!</span><span class="w"> </span><span class="n">collector</span><span class="w"> </span><span class="n">tuple</span><span class="p">)</span><span class="w">
+ </span><span class="p">)))))</span><span class="w">
+</span></code></pre></div>
<p>The implementation for a prepared bolt is a function that takes as input the topology config, <code>TopologyContext</code>, and <code>OutputCollector</code>, and returns an implementation of the <code>IBolt</code> interface. This design allows you to have a closure around the implementation of <code>execute</code> and <code>cleanup</code>. </p>
<p>In this example, the word counts are stored in the closure in a map called <code>counts</code>. The <code>bolt</code> macro is used to create the <code>IBolt</code> implementation. The <code>bolt</code> macro is a more concise way to implement the interface than reifying, and it automatically type-hints all of the method parameters. This bolt implements the execute method which updates the count in the map and emits the new word count.</p>
@@ -234,18 +234,18 @@
<h4 id="output-declarations">Output declarations</h4>
<p>The Clojure DSL has a concise syntax for declaring the outputs of a bolt. The most general way to declare the outputs is as a map from stream id a stream spec. For example:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">{</span><span class="s">"1"</span> <span class="p">[</span><span class="s">"field1"</span> <span class="s">"field2"</span><span class="p">]</span>
- <span class="s">"2"</span> <span class="p">(</span><span class="nf">direct-stream</span> <span class="p">[</span><span class="s">"f1"</span> <span class="s">"f2"</span> <span class="s">"f3"</span><span class="p">])</span>
- <span class="s">"3"</span> <span class="p">[</span><span class="s">"f1"</span><span class="p">]}</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">{</span><span class="s">"1"</span><span class="w"> </span><span class="p">[</span><span class="s">"field1"</span><span class="w"> </span><span class="s">"field2"</span><span class="p">]</span><span class="w">
+ </span><span class="s">"2"</span><span class="w"> </span><span class="p">(</span><span class="nf">direct-stream</span><span class="w"> </span><span class="p">[</span><span class="s">"f1"</span><span class="w"> </span><span class="s">"f2"</span><span class="w"> </span><span class="s">"f3"</span><span class="p">])</span><span class="w">
+ </span><span class="s">"3"</span><span class="w"> </span><span class="p">[</span><span class="s">"f1"</span><span class="p">]}</span><span class="w">
+</span></code></pre></div>
<p>The stream id is a string, while the stream spec is either a vector of fields or a vector of fields wrapped by <code>direct-stream</code>. <code>direct stream</code> marks the stream as a direct stream (See <a href="Concepts.html">Concepts</a> and <a href="">Direct groupings</a> for more details on direct streams).</p>
<p>If the bolt only has one output stream, you can define the default stream of the bolt by using a vector instead of a map for the output declaration. For example:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">[</span><span class="s">"word"</span> <span class="s">"count"</span><span class="p">]</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">[</span><span class="s">"word"</span><span class="w"> </span><span class="s">"count"</span><span class="p">]</span><span class="w">
+</span></code></pre></div>
<p>This declares the output of the bolt as the fields ["word" "count"] on the default stream id.</p>
-<h4 id="emitting,-acking,-and-failing">Emitting, acking, and failing</h4>
+<h4 id="emitting-acking-and-failing">Emitting, acking, and failing</h4>
<p>Rather than use the Java methods on <code>OutputCollector</code> directly, the DSL provides a nicer set of functions for using <code>OutputCollector</code>: <code>emit-bolt!</code>, <code>emit-direct-bolt!</code>, <code>ack!</code>, and <code>fail!</code>.</p>
@@ -269,23 +269,23 @@
<p>If you leave out the option map, it defaults to {:prepare true}. The output declaration for <code>defspout</code> has the same syntax as <code>defbolt</code>.</p>
<p>Here's an example <code>defspout</code> implementation from <a href="https://github.com/apache/storm/blob/master/examples/storm-starter/src/clj/storm/starter/clj/word_count.clj">storm-starter</a>:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defspout</span> <span class="nv">sentence-spout</span> <span class="p">[</span><span class="s">"sentence"</span><span class="p">]</span>
- <span class="p">[</span><span class="nv">conf</span> <span class="nv">context</span> <span class="nv">collector</span><span class="p">]</span>
- <span class="p">(</span><span class="k">let </span><span class="p">[</span><span class="nv">sentences</span> <span class="p">[</span><span class="s">"a little brown dog"</span>
- <span class="s">"the man petted the dog"</span>
- <span class="s">"four score and seven years ago"</span>
- <span class="s">"an apple a day keeps the doctor away"</span><span class="p">]]</span>
- <span class="p">(</span><span class="nf">spout</span>
- <span class="p">(</span><span class="nf">nextTuple</span> <span class="p">[]</span>
- <span class="p">(</span><span class="nf">Thread/sleep</span> <span class="mi">100</span><span class="p">)</span>
- <span class="p">(</span><span class="nf">emit-spout!</span> <span class="nv">collector</span> <span class="p">[(</span><span class="nf">rand-nth</span> <span class="nv">sentences</span><span class="p">)])</span>
- <span class="p">)</span>
- <span class="p">(</span><span class="nf">ack</span> <span class="p">[</span><span class="nv">id</span><span class="p">]</span>
- <span class="c1">;; You only need to define this method for reliable spouts</span>
- <span class="c1">;; (such as one that reads off of a queue like Kestrel)</span>
- <span class="c1">;; This is an unreliable spout, so it does nothing here</span>
- <span class="p">))))</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defspout</span><span class="w"> </span><span class="n">sentence-spout</span><span class="w"> </span><span class="p">[</span><span class="s">"sentence"</span><span class="p">]</span><span class="w">
+ </span><span class="p">[</span><span class="n">conf</span><span class="w"> </span><span class="n">context</span><span class="w"> </span><span class="n">collector</span><span class="p">]</span><span class="w">
+ </span><span class="p">(</span><span class="k">let</span><span class="w"> </span><span class="p">[</span><span class="n">sentences</span><span class="w"> </span><span class="p">[</span><span class="s">"a little brown dog"</span><span class="w">
+ </span><span class="s">"the man petted the dog"</span><span class="w">
+ </span><span class="s">"four score and seven years ago"</span><span class="w">
+ </span><span class="s">"an apple a day keeps the doctor away"</span><span class="p">]]</span><span class="w">
+ </span><span class="p">(</span><span class="nf">spout</span><span class="w">
+ </span><span class="p">(</span><span class="nf">nextTuple</span><span class="w"> </span><span class="p">[]</span><span class="w">
+ </span><span class="p">(</span><span class="nf">Thread/sleep</span><span class="w"> </span><span class="mi">100</span><span class="p">)</span><span class="w">
+ </span><span class="p">(</span><span class="nf">emit-spout!</span><span class="w"> </span><span class="n">collector</span><span class="w"> </span><span class="p">[(</span><span class="nf">rand-nth</span><span class="w"> </span><span class="n">sentences</span><span class="p">)])</span><span class="w">
+ </span><span class="p">)</span><span class="w">
+ </span><span class="p">(</span><span class="nf">ack</span><span class="w"> </span><span class="p">[</span><span class="n">id</span><span class="p">]</span><span class="w">
+ </span><span class="c1">;; You only need to define this method for reliable spouts
+</span><span class="w"> </span><span class="c1">;; (such as one that reads off of a queue like Kestrel)
+</span><span class="w"> </span><span class="c1">;; This is an unreliable spout, so it does nothing here
+</span><span class="w"> </span><span class="p">))))</span><span class="w">
+</span></code></pre></div>
<p>The implementation takes in as input the topology config, <code>TopologyContext</code>, and <code>SpoutOutputCollector</code>. The implementation returns an <code>ISpout</code> object. Here, the <code>nextTuple</code> function emits a random sentence from <code>sentences</code>. </p>
<p>This spout isn't reliable, so the <code>ack</code> and <code>fail</code> methods will never be called. A reliable spout will add a message id when emitting tuples, and then <code>ack</code> or <code>fail</code> will be called when the tuple is completed or failed respectively. See <a href="Guaranteeing-message-processing.html">Guaranteeing message processing</a> for more info on how reliability works within Storm.</p>
@@ -295,25 +295,25 @@
<p>There is also a <code>emit-direct-spout!</code> function that emits a tuple to a direct stream and takes an additional argument as the second parameter of the task id to send the tuple to.</p>
<p>Spouts can be parameterized just like bolts, in which case the symbol is bound to a function returning <code>IRichSpout</code> instead of the <code>IRichSpout</code> itself. You can also declare an unprepared spout which only defines the <code>nextTuple</code> method. Here is an example of an unprepared spout that emits random sentences parameterized at runtime:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defspout</span> <span class="nv">sentence-spout-parameterized</span> <span class="p">[</span><span class="s">"word"</span><span class="p">]</span> <span class="p">{</span><span class="ss">:params</span> <span class="p">[</span><span class="nv">sentences</span><span class="p">]</span> <span class="ss">:prepare</span> <span class="nv">false</span><span class="p">}</span>
- <span class="p">[</span><span class="nv">collector</span><span class="p">]</span>
- <span class="p">(</span><span class="nf">Thread/sleep</span> <span class="mi">500</span><span class="p">)</span>
- <span class="p">(</span><span class="nf">emit-spout!</span> <span class="nv">collector</span> <span class="p">[(</span><span class="nf">rand-nth</span> <span class="nv">sentences</span><span class="p">)]))</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">defspout</span><span class="w"> </span><span class="n">sentence-spout-parameterized</span><span class="w"> </span><span class="p">[</span><span class="s">"word"</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="no">:params</span><span class="w"> </span><span class="p">[</span><span class="n">sentences</span><span class="p">]</span><span class="w"> </span><span class="no">:prepare</span><span class="w"> </span><span class="n">false</span><span class="p">}</span><span class="w">
+ </span><span class="p">[</span><span class="n">collector</span><span class="p">]</span><span class="w">
+ </span><span class="p">(</span><span class="nf">Thread/sleep</span><span class="w"> </span><span class="mi">500</span><span class="p">)</span><span class="w">
+ </span><span class="p">(</span><span class="nf">emit-spout!</span><span class="w"> </span><span class="n">collector</span><span class="w"> </span><span class="p">[(</span><span class="nf">rand-nth</span><span class="w"> </span><span class="n">sentences</span><span class="p">)]))</span><span class="w">
+</span></code></pre></div>
<p>The following example illustrates how to use this spout in a <code>spout-spec</code>:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">spout-spec</span> <span class="p">(</span><span class="nf">sentence-spout-parameterized</span>
- <span class="p">[</span><span class="s">"the cat jumped over the door"</span>
- <span class="s">"greetings from a faraway land"</span><span class="p">])</span>
- <span class="ss">:p</span> <span class="mi">2</span><span class="p">)</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">(</span><span class="nf">spout-spec</span><span class="w"> </span><span class="p">(</span><span class="nf">sentence-spout-parameterized</span><span class="w">
+ </span><span class="p">[</span><span class="s">"the cat jumped over the door"</span><span class="w">
+ </span><span class="s">"greetings from a faraway land"</span><span class="p">])</span><span class="w">
+ </span><span class="no">:p</span><span class="w"> </span><span class="mi">2</span><span class="p">)</span><span class="w">
+</span></code></pre></div>
<h3 id="running-topologies-in-local-mode-or-on-a-cluster">Running topologies in local mode or on a cluster</h3>
<p>That's all there is to the Clojure DSL. To submit topologies in remote mode or local mode, just use the <code>StormSubmitter</code> or <code>LocalCluster</code> classes just like you would from Java.</p>
<p>To create topology configs, it's easiest to use the <code>backtype.storm.config</code> namespace which defines constants for all of the possible configs. The constants are the same as the static constants in the <code>Config</code> class, except with dashes instead of underscores. For example, here's a topology config that sets the number of workers to 15 and configures the topology in debug mode:</p>
-<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">{</span><span class="nv">TOPOLOGY-DEBUG</span> <span class="nv">true</span>
- <span class="nv">TOPOLOGY-WORKERS</span> <span class="mi">15</span><span class="p">}</span>
-</code></pre></div>
+<div class="highlight"><pre><code class="language-clojure" data-lang="clojure"><span class="p">{</span><span class="n">TOPOLOGY-DEBUG</span><span class="w"> </span><span class="n">true</span><span class="w">
+ </span><span class="n">TOPOLOGY-WORKERS</span><span class="w"> </span><span class="mi">15</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<h3 id="testing-topologies">Testing topologies</h3>
<p><a href="http://www.pixelmachine.org/2011/12/17/Testing-Storm-Topologies.html">This blog post</a> and its <a href="http://www.pixelmachine.org/2011/12/21/Testing-Storm-Topologies-Part-2.html">follow-up</a> give a good overview of Storm's powerful built-in facilities for testing topologies in Clojure.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Common-patterns.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Common-patterns.html b/_site/documentation/Common-patterns.html
index 79d35f0..3a4a7ff 100644
--- a/_site/documentation/Common-patterns.html
+++ b/_site/documentation/Common-patterns.html
@@ -107,10 +107,10 @@
<p>A streaming join combines two or more data streams together based on some common field. Whereas a normal database join has finite input and clear semantics for a join, a streaming join has infinite input and unclear semantics for what a join should be.</p>
<p>The join type you need will vary per application. Some applications join all tuples for two streams over a finite window of time, whereas other applications expect exactly one tuple for each side of the join for each join field. Other applications may do the join completely differently. The common pattern among all these join types is partitioning multiple input streams in the same way. This is easily accomplished in Storm by using a fields grouping on the same fields for many input streams to the joiner bolt. For example:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"join"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">MyJoiner</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"1"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"joinfield1"</span><span class="o">,</span> <span class="s">"joinfield2"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"2"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"joinfield1"</span><span class="o">,</span> <span class="s">"joinfield2"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"3"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"joinfield1"</span><span class="o">,</span> <span class="s">"joinfield2"</span><span class="o">));</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"join"</span><span class="o">,</span> <span class="k">new</span> <span class="n">MyJoiner</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"1"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"joinfield1"</span><span class="o">,</span> <span class="s">"joinfield2"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"2"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"joinfield1"</span><span class="o">,</span> <span class="s">"joinfield2"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"3"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"joinfield1"</span><span class="o">,</span> <span class="s">"joinfield2"</span><span class="o">));</span>
</code></pre></div>
<p>The different streams don't have to have the same field names, of course.</p>
@@ -126,13 +126,13 @@
<p>Many bolts follow a similar pattern of reading an input tuple, emitting zero or more tuples based on that input tuple, and then acking that input tuple immediately at the end of the execute method. Bolts that match this pattern are things like functions and filters. This is such a common pattern that Storm exposes an interface called <a href="/javadoc/apidocs/backtype/storm/topology/IBasicBolt.html">IBasicBolt</a> that automates this pattern for you. See <a href="Guaranteeing-message-processing.html">Guaranteeing message processing</a> for more information.</p>
-<h3 id="in-memory-caching-+-fields-grouping-combo">In-memory caching + fields grouping combo</h3>
+<h3 id="in-memory-caching-fields-grouping-combo">In-memory caching + fields grouping combo</h3>
<p>It's common to keep caches in-memory in Storm bolts. Caching becomes particularly powerful when you combine it with a fields grouping. For example, suppose you have a bolt that expands short URLs (like bit.ly, t.co, etc.) into long URLs. You can increase performance by keeping an LRU cache of short URL to long URL expansions to avoid doing the same HTTP requests over and over. Suppose component "urls" emits short URLS, and component "expand" expands short URLs into long URLs and keeps a cache internally. Consider the difference between the two following snippets of code:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"expand"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">ExpandUrl</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"expand"</span><span class="o">,</span> <span class="k">new</span> <span class="n">ExpandUrl</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
<span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="mi">1</span><span class="o">);</span>
-</code></pre></div><div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"expand"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">ExpandUrl</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"urls"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"url"</span><span class="o">));</span>
+</code></pre></div><div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"expand"</span><span class="o">,</span> <span class="k">new</span> <span class="n">ExpandUrl</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"urls"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"url"</span><span class="o">));</span>
</code></pre></div>
<p>The second approach will have vastly more effective caches, since the same URL will always go to the same task. This avoids having duplication across any of the caches in the tasks and makes it much more likely that a short URL will hit the cache.</p>
@@ -141,20 +141,20 @@
<p>A common continuous computation done on Storm is a "streaming top N" of some sort. Suppose you have a bolt that emits tuples of the form ["value", "count"] and you want a bolt that emits the top N tuples based on count. The simplest way to do this is to have a bolt that does a global grouping on the stream and maintains a list in memory of the top N items.</p>
<p>This approach obviously doesn't scale to large streams since the entire stream has to go through one task. A better way to do the computation is to do many top N's in parallel across partitions of the stream, and then merge those top N's together to get the global top N. The pattern looks like this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"rank"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">RankObjects</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"objects"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"value"</span><span class="o">));</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"merge"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">MergeObjects</span><span class="o">())</span>
- <span class="o">.</span><span class="na">globalGrouping</span><span class="o">(</span><span class="s">"rank"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"rank"</span><span class="o">,</span> <span class="k">new</span> <span class="n">RankObjects</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"objects"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"value"</span><span class="o">));</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"merge"</span><span class="o">,</span> <span class="k">new</span> <span class="n">MergeObjects</span><span class="o">())</span>
+ <span class="o">.</span><span class="na">globalGrouping</span><span class="o">(</span><span class="s">"rank"</span><span class="o">);</span>
</code></pre></div>
<p>This pattern works because of the fields grouping done by the first bolt which gives the partitioning you need for this to be semantically correct. You can see an example of this pattern in storm-starter <a href="https://github.com/apache/storm/blob/master/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java">here</a>.</p>
<p>If however you have a known skew in the data being processed it can be advantageous to use partialKeyGrouping instead of fieldsGrouping. This will distribute the load for each key between two downstream bolts instead of a single one.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"count"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">CountObjects</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
- <span class="o">.</span><span class="na">partialKeyGrouping</span><span class="o">(</span><span class="s">"objects"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"value"</span><span class="o">));</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"rank"</span> <span class="k">new</span> <span class="nf">AggregateCountsAndRank</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
- <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"count"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"key"</span><span class="o">))</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"merge"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">MergeRanksObjects</span><span class="o">())</span>
- <span class="o">.</span><span class="na">globalGrouping</span><span class="o">(</span><span class="s">"rank"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"count"</span><span class="o">,</span> <span class="k">new</span> <span class="n">CountObjects</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">partialKeyGrouping</span><span class="o">(</span><span class="s">"objects"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"value"</span><span class="o">));</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"rank"</span> <span class="k">new</span> <span class="n">AggregateCountsAndRank</span><span class="o">(),</span> <span class="n">parallelism</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"count"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"key"</span><span class="o">))</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"merge"</span><span class="o">,</span> <span class="k">new</span> <span class="n">MergeRanksObjects</span><span class="o">())</span>
+ <span class="o">.</span><span class="na">globalGrouping</span><span class="o">(</span><span class="s">"rank"</span><span class="o">);</span>
</code></pre></div>
<p>The topology needs an extra layer of processing to aggregate the partial counts from the upstream bolts but this only processes aggregated values now so the bolt it is not subject to the load caused by the skewed data. You can see an example of this pattern in storm-starter <a href="https://github.com/apache/storm/blob/master/examples/storm-starter/src/jvm/storm/starter/SkewedRollingTopWords.java">here</a>.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Creating-a-new-Storm-project.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Creating-a-new-Storm-project.html b/_site/documentation/Creating-a-new-Storm-project.html
index 663852e..7007612 100644
--- a/_site/documentation/Creating-a-new-Storm-project.html
+++ b/_site/documentation/Creating-a-new-Storm-project.html
@@ -105,7 +105,7 @@
<p>To set up the classpath in Eclipse, create a new Java project, include <code>src/jvm/</code> as a source path, and make sure all the jars in <code>lib/</code> and <code>lib/dev/</code> are in the <code>Referenced Libraries</code> section of the project.</p>
-<h3 id="if-using-multilang,-add-multilang-dir-to-classpath">If using multilang, add multilang dir to classpath</h3>
+<h3 id="if-using-multilang-add-multilang-dir-to-classpath">If using multilang, add multilang dir to classpath</h3>
<p>If you implement spouts or bolts in languages other than Java, then those implementations should be under the <code>multilang/resources/</code> directory of the project. For Storm to find these files in local mode, the <code>resources/</code> dir needs to be on the classpath. You can do this in Eclipse by adding <code>multilang/</code> as a source folder. You may also need to add multilang/resources as a source directory.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Defining-a-non-jvm-language-dsl-for-storm.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Defining-a-non-jvm-language-dsl-for-storm.html b/_site/documentation/Defining-a-non-jvm-language-dsl-for-storm.html
index 0f32d3c..1621534 100644
--- a/_site/documentation/Defining-a-non-jvm-language-dsl-for-storm.html
+++ b/_site/documentation/Defining-a-non-jvm-language-dsl-for-storm.html
@@ -93,7 +93,7 @@
<p>The right place to start to learn how to make a non-JVM DSL for Storm is <a href="https://github.com/apache/storm/blob/master/storm-core/src/storm.thrift">storm-core/src/storm.thrift</a>. Since Storm topologies are just Thrift structures, and Nimbus is a Thrift daemon, you can create and submit topologies in any language.</p>
<p>When you create the Thrift structs for spouts and bolts, the code for the spout or bolt is specified in the ComponentObject struct:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">union ComponentObject {
+<div class="highlight"><pre><code class="language-" data-lang="">union ComponentObject {
1: binary serialized_java;
2: ShellComponent shell;
3: JavaObject java_object;
@@ -102,13 +102,13 @@
<p>For a Python DSL, you would want to make use of "2" and "3". ShellComponent lets you specify a script to run that component (e.g., your python code). And JavaObject lets you specify native java spouts and bolts for the component (and Storm will use reflection to create that spout or bolt).</p>
<p>There's a "storm shell" command that will help with submitting a topology. Its usage is like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">storm shell resources/ python topology.py arg1 arg2
+<div class="highlight"><pre><code class="language-" data-lang="">storm shell resources/ python topology.py arg1 arg2
</code></pre></div>
<p>storm shell will then package resources/ into a jar, upload the jar to Nimbus, and call your topology.py script like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">python topology.py arg1 arg2 {nimbus-host} {nimbus-port} {uploaded-jar-location}
+<div class="highlight"><pre><code class="language-" data-lang="">python topology.py arg1 arg2 {nimbus-host} {nimbus-port} {uploaded-jar-location}
</code></pre></div>
<p>Then you can connect to Nimbus using the Thrift API and submit the topology, passing {uploaded-jar-location} into the submitTopology method. For reference, here's the submitTopology definition:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kt">void</span> <span class="nf">submitTopology</span><span class="o">(</span><span class="mi">1</span><span class="o">:</span> <span class="n">string</span> <span class="n">name</span><span class="o">,</span> <span class="mi">2</span><span class="o">:</span> <span class="n">string</span> <span class="n">uploadedJarLocation</span><span class="o">,</span> <span class="mi">3</span><span class="o">:</span> <span class="n">string</span> <span class="n">jsonConf</span><span class="o">,</span> <span class="mi">4</span><span class="o">:</span> <span class="n">StormTopology</span> <span class="n">topology</span><span class="o">)</span> <span class="kd">throws</span> <span class="o">(</span><span class="mi">1</span><span class="o">:</span> <span class="n">AlreadyAliveException</span> <span class="n">e</span><span class="o">,</span> <span class="mi">2</span><span class="o">:</span> <span class="n">InvalidTop
ologyException</span> <span class="n">ite</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kt">void</span> <span class="nf">submitTopology</span><span class="p">(</span><span class="mi">1</span><span class="o">:</span> <span class="n">string</span> <span class="n">name</span><span class="o">,</span> <span class="mi">2</span><span class="o">:</span> <span class="n">string</span> <span class="n">uploadedJarLocation</span><span class="o">,</span> <span class="mi">3</span><span class="o">:</span> <span class="n">string</span> <span class="n">jsonConf</span><span class="o">,</span> <span class="mi">4</span><span class="o">:</span> <span class="n">StormTopology</span> <span class="n">topology</span><span class="o">)</span> <span class="kd">throws</span> <span class="o">(</span><span class="mi">1</span><span class="o">:</span> <span class="n">AlreadyAliveException</span> <span class="n">e</span><span class="o">,</span> <span class="mi">2</span><span class="o">:</span> <span class="n">InvalidTop
ologyException</span> <span class="n">ite</span><span class="o">);</span>
</code></pre></div>
<p>Finally, one of the key things to do in a non-JVM DSL is make it easy to define the entire topology in one file (the bolts, spouts, and the definition of the topology).</p>
[06/10] storm git commit: STORM-1366. Add documentation for StormSQL
into the website.
Posted by wh...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Trident-API-Overview.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Trident-API-Overview.html b/_site/documentation/Trident-API-Overview.html
index 961714d..f3e883e 100644
--- a/_site/documentation/Trident-API-Overview.html
+++ b/_site/documentation/Trident-API-Overview.html
@@ -110,23 +110,23 @@
<p>A function takes in a set of input fields and emits zero or more tuples as output. The fields of the output tuple are appended to the original input tuple in the stream. If a function emits no tuples, the original input tuple is filtered out. Otherwise, the input tuple is duplicated for each output tuple. Suppose you have this function:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">MyFunction</span> <span class="kd">extends</span> <span class="n">BaseFunction</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="k">for</span><span class="o">(</span><span class="kt">int</span> <span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="o">;</span> <span class="n">i</span> <span class="o"><</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getInteger</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span> <span class="n">i</span><span class="o">++)</span> <span class="o">{</span>
- <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">i</span><span class="o">));</span>
+ <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">i</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
<p>Now suppose you have a stream in the variable "mystream" with the fields ["a", "b", "c"] with the following tuples:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">[1, 2, 3]
+<div class="highlight"><pre><code class="language-" data-lang="">[1, 2, 3]
[4, 1, 6]
[3, 0, 8]
</code></pre></div>
<p>If you run this code:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">MyFunction</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"d"</span><span class="o">)))</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">),</span> <span class="k">new</span> <span class="n">MyFunction</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"d"</span><span class="o">)))</span>
</code></pre></div>
<p>The resulting tuples would have fields ["a", "b", "c", "d"] and look like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">[1, 2, 3, 0]
+<div class="highlight"><pre><code class="language-" data-lang="">[1, 2, 3, 0]
[1, 2, 3, 1]
[4, 1, 6, 0]
</code></pre></div>
@@ -134,43 +134,43 @@
<p>Filters take in a tuple as input and decide whether or not to keep that tuple or not. Suppose you had this filter:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">MyFilter</span> <span class="kd">extends</span> <span class="n">BaseFilter</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">boolean</span> <span class="nf">isKeep</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">boolean</span> <span class="n">isKeep</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="k">return</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getInteger</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="o">&&</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getInteger</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> <span class="o">==</span> <span class="mi">2</span><span class="o">;</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
<p>Now suppose you had these tuples with fields ["a", "b", "c"]:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">[1, 2, 3]
+<div class="highlight"><pre><code class="language-" data-lang="">[1, 2, 3]
[2, 1, 1]
[2, 3, 4]
</code></pre></div>
<p>If you ran this code:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">,</span> <span class="s">"a"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">MyFilter</span><span class="o">())</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">,</span> <span class="s">"a"</span><span class="o">),</span> <span class="k">new</span> <span class="n">MyFilter</span><span class="o">())</span>
</code></pre></div>
<p>The resulting tuples would be:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">[2, 1, 1]
+<div class="highlight"><pre><code class="language-" data-lang="">[2, 1, 1]
</code></pre></div>
<h3 id="partitionaggregate">partitionAggregate</h3>
<p>partitionAggregate runs a function on each partition of a batch of tuples. Unlike functions, the tuples emitted by partitionAggregate replace the input tuples given to it. Consider this example:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">partitionAggregate</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">))</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">partitionAggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">))</span>
</code></pre></div>
<p>Suppose the input stream contained fields ["a", "b"] and the following partitions of tuples:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">Partition 0:
-["a", 1]
-["b", 2]
+<div class="highlight"><pre><code class="language-" data-lang="">Partition 0:
+["a", 1]
+["b", 2]
Partition 1:
-["a", 3]
-["c", 8]
+["a", 3]
+["c", 8]
Partition 2:
-["e", 1]
-["d", 9]
-["d", 10]
+["e", 1]
+["d", 9]
+["d", 10]
</code></pre></div>
<p>Then the output stream of that code would contain these tuples with one field called "sum":</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">Partition 0:
+<div class="highlight"><pre><code class="language-" data-lang="">Partition 0:
[3]
Partition 1:
@@ -183,22 +183,22 @@ Partition 2:
<p>Here's the interface for CombinerAggregator:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">CombinerAggregator</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="n">T</span> <span class="nf">init</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">);</span>
- <span class="n">T</span> <span class="nf">combine</span><span class="o">(</span><span class="n">T</span> <span class="n">val1</span><span class="o">,</span> <span class="n">T</span> <span class="n">val2</span><span class="o">);</span>
- <span class="n">T</span> <span class="nf">zero</span><span class="o">();</span>
+ <span class="n">T</span> <span class="n">init</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="n">T</span> <span class="n">combine</span><span class="o">(</span><span class="n">T</span> <span class="n">val1</span><span class="o">,</span> <span class="n">T</span> <span class="n">val2</span><span class="o">);</span>
+ <span class="n">T</span> <span class="n">zero</span><span class="o">();</span>
<span class="o">}</span>
</code></pre></div>
<p>A CombinerAggregator returns a single tuple with a single field as output. CombinerAggregators run the init function on each input tuple and use the combine function to combine values until there's only one value left. If there's no tuples in the partition, the CombinerAggregator emits the output of the zero function. For example, here's the implementation of Count:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">Count</span> <span class="kd">implements</span> <span class="n">CombinerAggregator</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="o">{</span>
- <span class="kd">public</span> <span class="n">Long</span> <span class="nf">init</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">Long</span> <span class="n">init</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="k">return</span> <span class="mi">1L</span><span class="o">;</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="n">Long</span> <span class="nf">combine</span><span class="o">(</span><span class="n">Long</span> <span class="n">val1</span><span class="o">,</span> <span class="n">Long</span> <span class="n">val2</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">Long</span> <span class="n">combine</span><span class="o">(</span><span class="n">Long</span> <span class="n">val1</span><span class="o">,</span> <span class="n">Long</span> <span class="n">val2</span><span class="o">)</span> <span class="o">{</span>
<span class="k">return</span> <span class="n">val1</span> <span class="o">+</span> <span class="n">val2</span><span class="o">;</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="n">Long</span> <span class="nf">zero</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">Long</span> <span class="n">zero</span><span class="o">()</span> <span class="o">{</span>
<span class="k">return</span> <span class="mi">0L</span><span class="o">;</span>
<span class="o">}</span>
<span class="o">}</span>
@@ -207,17 +207,17 @@ Partition 2:
<p>A ReducerAggregator has the following interface:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">ReducerAggregator</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="n">T</span> <span class="nf">init</span><span class="o">();</span>
- <span class="n">T</span> <span class="nf">reduce</span><span class="o">(</span><span class="n">T</span> <span class="n">curr</span><span class="o">,</span> <span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="n">T</span> <span class="n">init</span><span class="o">();</span>
+ <span class="n">T</span> <span class="n">reduce</span><span class="o">(</span><span class="n">T</span> <span class="n">curr</span><span class="o">,</span> <span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>A ReducerAggregator produces an initial value with init, and then it iterates on that value for each input tuple to produce a single tuple with a single value as output. For example, here's how you would define Count as a ReducerAggregator:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">Count</span> <span class="kd">implements</span> <span class="n">ReducerAggregator</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="o">{</span>
- <span class="kd">public</span> <span class="n">Long</span> <span class="nf">init</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">Long</span> <span class="n">init</span><span class="o">()</span> <span class="o">{</span>
<span class="k">return</span> <span class="mi">0L</span><span class="o">;</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="n">Long</span> <span class="nf">reduce</span><span class="o">(</span><span class="n">Long</span> <span class="n">curr</span><span class="o">,</span> <span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">Long</span> <span class="n">reduce</span><span class="o">(</span><span class="n">Long</span> <span class="n">curr</span><span class="o">,</span> <span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="k">return</span> <span class="n">curr</span> <span class="o">+</span> <span class="mi">1</span><span class="o">;</span>
<span class="o">}</span>
<span class="o">}</span>
@@ -226,9 +226,9 @@ Partition 2:
<p>The most general interface for performing aggregations is Aggregator, which looks like this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">Aggregator</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="kd">extends</span> <span class="n">Operation</span> <span class="o">{</span>
- <span class="n">T</span> <span class="nf">init</span><span class="o">(</span><span class="n">Object</span> <span class="n">batchId</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">);</span>
- <span class="kt">void</span> <span class="nf">aggregate</span><span class="o">(</span><span class="n">T</span> <span class="n">state</span><span class="o">,</span> <span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">);</span>
- <span class="kt">void</span> <span class="nf">complete</span><span class="o">(</span><span class="n">T</span> <span class="n">state</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">);</span>
+ <span class="n">T</span> <span class="n">init</span><span class="o">(</span><span class="n">Object</span> <span class="n">batchId</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">aggregate</span><span class="o">(</span><span class="n">T</span> <span class="n">state</span><span class="o">,</span> <span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">complete</span><span class="o">(</span><span class="n">T</span> <span class="n">state</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>Aggregators can emit any number of tuples with any number of fields. They can emit tuples at any point during execution. Aggregators execute in the following way:</p>
@@ -245,23 +245,23 @@ Partition 2:
<span class="kt">long</span> <span class="n">count</span> <span class="o">=</span> <span class="mi">0</span><span class="o">;</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="n">CountState</span> <span class="nf">init</span><span class="o">(</span><span class="n">Object</span> <span class="n">batchId</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
- <span class="k">return</span> <span class="k">new</span> <span class="nf">CountState</span><span class="o">();</span>
+ <span class="kd">public</span> <span class="n">CountState</span> <span class="n">init</span><span class="o">(</span><span class="n">Object</span> <span class="n">batchId</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="k">return</span> <span class="k">new</span> <span class="n">CountState</span><span class="o">();</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">aggregate</span><span class="o">(</span><span class="n">CountState</span> <span class="n">state</span><span class="o">,</span> <span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">aggregate</span><span class="o">(</span><span class="n">CountState</span> <span class="n">state</span><span class="o">,</span> <span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="n">state</span><span class="o">.</span><span class="na">count</span><span class="o">+=</span><span class="mi">1</span><span class="o">;</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">complete</span><span class="o">(</span><span class="n">CountState</span> <span class="n">state</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">state</span><span class="o">.</span><span class="na">count</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">complete</span><span class="o">(</span><span class="n">CountState</span> <span class="n">state</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">state</span><span class="o">.</span><span class="na">count</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
<p>Sometimes you want to execute multiple aggregators at the same time. This is called chaining and can be accomplished like this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">chainedAgg</span><span class="o">()</span>
- <span class="o">.</span><span class="na">partitionAggregate</span><span class="o">(</span><span class="k">new</span> <span class="nf">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">partitionAggregate</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">partitionAggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">partitionAggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Sum</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"sum"</span><span class="o">))</span>
<span class="o">.</span><span class="na">chainEnd</span><span class="o">()</span>
</code></pre></div>
<p>This code will run the Count and Sum aggregators on each partition. The output will contain a single tuple with the fields ["count", "sum"].</p>
@@ -273,7 +273,7 @@ Partition 2:
<h3 id="projection">projection</h3>
<p>The projection method on Stream keeps only the fields specified in the operation. If you had a Stream with fields ["a", "b", "c", "d"] and you ran this code:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">project</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">,</span> <span class="s">"d"</span><span class="o">))</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">project</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"b"</span><span class="o">,</span> <span class="s">"d"</span><span class="o">))</span>
</code></pre></div>
<p>The output stream would contain only the fields ["b", "d"].</p>
@@ -297,7 +297,7 @@ Partition 2:
<p>Running aggregate on a Stream does a global aggregation. When you use a ReducerAggregator or an Aggregator, the stream is first repartitioned into a single partition, and then the aggregation function is run on that partition. When you use a CombinerAggregator, on the other hand, first Trident will compute partial aggregations of each partition, then repartition to a single partition, and then finish the aggregation after the network transfer. CombinerAggregator's are far more efficient and should be used when possible.</p>
<p>Here's an example of using aggregate to get a global count for a batch:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="nf">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">mystream</span><span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
</code></pre></div>
<p>Like partitionAggregate, aggregators for aggregate can be chained. However, if you chain a CombinerAggregator with a non-CombinerAggregator, Trident is unable to do the partial aggregation optimization.</p>
@@ -323,7 +323,7 @@ Partition 2:
<p>Another way to combine streams is with a join. Now, a standard join, like the kind from SQL, require finite input. So they don't make sense with infinite streams. Joins in Trident only apply within each small batch that comes off of the spout. </p>
<p>Here's an example join between a stream containing fields ["key", "val1", "val2"] and another stream containing ["x", "val1"]:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">topology</span><span class="o">.</span><span class="na">join</span><span class="o">(</span><span class="n">stream1</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"key"</span><span class="o">),</span> <span class="n">stream2</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"x"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"key"</span><span class="o">,</span> <span class="s">"a"</span><span class="o">,</span> <span class="s">"b"</span><span class="o">,</span> <span class="s">"c"</span><span class="o">));</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">topology</span><span class="o">.</span><span class="na">join</span><span class="o">(</span><span class="n">stream1</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"key"</span><span class="o">),</span> <span class="n">stream2</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"x"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"key"</span><span class="o">,</span> <span class="s">"a"</span><span class="o">,</span> <span class="s">"b"</span><span class="o">,</span> <span class="s">"c"</span><span class="o">));</span>
</code></pre></div>
<p>This joins stream1 and stream2 together using "key" and "x" as the join fields for each respective stream. Then, Trident requires that all the output fields of the new stream be named, since the input streams could have overlapping field names. The tuples emitted from the join will contain:</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Trident-spouts.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Trident-spouts.html b/_site/documentation/Trident-spouts.html
index f4388eb..200c145 100644
--- a/_site/documentation/Trident-spouts.html
+++ b/_site/documentation/Trident-spouts.html
@@ -97,8 +97,8 @@
<p>There is an inextricable link between how you source your data streams and how you update state (e.g. databases) based on those data streams. See <a href="Trident-state.html">Trident state doc</a> for an explanation of this – understanding this link is imperative for understanding the spout options available.</p>
<p>Regular Storm spouts will be non-transactional spouts in a Trident topology. To use a regular Storm IRichSpout, create the stream like this in a TridentTopology:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TridentTopology</span><span class="o">();</span>
-<span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"myspoutid"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">MyRichSpout</span><span class="o">());</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TridentTopology</span><span class="o">();</span>
+<span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"myspoutid"</span><span class="o">,</span> <span class="k">new</span> <span class="n">MyRichSpout</span><span class="o">());</span>
</code></pre></div>
<p>All spouts in a Trident topology are required to be given a unique identifier for the stream – this identifier must be unique across all topologies run on the cluster. Trident will use this identifier to store metadata about what the spout has consumed in Zookeeper, including the txid and any metadata associated with the spout.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Trident-state.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Trident-state.html b/_site/documentation/Trident-state.html
index b6df40e..cd79194 100644
--- a/_site/documentation/Trident-state.html
+++ b/_site/documentation/Trident-state.html
@@ -131,17 +131,17 @@
<p>Suppose your topology computes word count and you want to store the word counts in a key/value database. The key will be the word, and the value will contain the count. You've already seen that storing just the count as the value isn't sufficient to know whether you've processed a batch of tuples before. Instead, what you can do is store the transaction id with the count in the database as an atomic value. Then, when updating the count, you can just compare the transaction id in the database with the transaction id for the current batch. If they're the same, you skip the update – because of the strong ordering, you know for sure that the value in the database incorporates the current batch. If they're different, you increment the count. This logic works because the batch for a txid never changes, and Trident ensures that state updates are ordered among batches.</p>
<p>Consider this example of why it works. Suppose you are processing txid 3 which consists of the following batch of tuples:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">["man"]
-["man"]
-["dog"]
+<div class="highlight"><pre><code class="language-" data-lang="">["man"]
+["man"]
+["dog"]
</code></pre></div>
<p>Suppose the database currently holds the following key/value pairs:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">man => [count=3, txid=1]
+<div class="highlight"><pre><code class="language-" data-lang="">man => [count=3, txid=1]
dog => [count=4, txid=3]
apple => [count=10, txid=2]
</code></pre></div>
<p>The txid associated with "man" is txid 1. Since the current txid is 3, you know for sure that this batch of tuples is not represented in that count. So you can go ahead and increment the count by 2 and update the txid. On the other hand, the txid for "dog" is the same as the current txid. So you know for sure that the increment from the current batch is already represented in the database for the "dog" key. So you can skip the update. After completing updates, the database looks like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">man => [count=5, txid=3]
+<div class="highlight"><pre><code class="language-" data-lang="">man => [count=5, txid=3]
dog => [count=4, txid=3]
apple => [count=10, txid=2]
</code></pre></div>
@@ -160,23 +160,23 @@ apple => [count=10, txid=2]
<p>With opaque transactional spouts, it's no longer possible to use the trick of skipping state updates if the transaction id in the database is the same as the transaction id for the current batch. This is because the batch may have changed between state updates.</p>
<p>What you can do is store more state in the database. Rather than store a value and transaction id in the database, you instead store a value, transaction id, and the previous value in the database. Let's again use the example of storing a count in the database. Suppose the partial count for your batch is "2" and it's time to apply a state update. Suppose the value in the database looks like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{ value = 4,
- prevValue = 1,
- txid = 2
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w"> </span><span class="err">value</span><span class="w"> </span><span class="err">=</span><span class="w"> </span><span class="err">4,</span><span class="w">
+ </span><span class="err">prevValue</span><span class="w"> </span><span class="err">=</span><span class="w"> </span><span class="err">1,</span><span class="w">
+ </span><span class="err">txid</span><span class="w"> </span><span class="err">=</span><span class="w"> </span><span class="err">2</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>Suppose your current txid is 3, different than what's in the database. In this case, you set "prevValue" equal to "value", increment "value" by your partial count, and update the txid. The new database value will look like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{ value = 6,
- prevValue = 4,
- txid = 3
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w"> </span><span class="err">value</span><span class="w"> </span><span class="err">=</span><span class="w"> </span><span class="err">6,</span><span class="w">
+ </span><span class="err">prevValue</span><span class="w"> </span><span class="err">=</span><span class="w"> </span><span class="err">4,</span><span class="w">
+ </span><span class="err">txid</span><span class="w"> </span><span class="err">=</span><span class="w"> </span><span class="err">3</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>Now suppose your current txid is 2, equal to what's in the database. Now you know that the "value" in the database contains an update from a previous batch for your current txid, but that batch may have been different so you have to ignore it. What you do in this case is increment "prevValue" by your partial count to compute the new "value". You then set the value in the database to this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{ value = 3,
- prevValue = 1,
- txid = 2
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w"> </span><span class="err">value</span><span class="w"> </span><span class="err">=</span><span class="w"> </span><span class="err">3,</span><span class="w">
+ </span><span class="err">prevValue</span><span class="w"> </span><span class="err">=</span><span class="w"> </span><span class="err">1,</span><span class="w">
+ </span><span class="err">txid</span><span class="w"> </span><span class="err">=</span><span class="w"> </span><span class="err">2</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>This works because of the strong ordering of batches provided by Trident. Once Trident moves onto a new batch for state updates, it will never go back to a previous batch. And since opaque transactional spouts guarantee no overlap between batches – that each tuple is successfully processed by one batch – you can safely update based on the previous value.</p>
<h2 id="non-transactional-spouts">Non-transactional spouts</h2>
@@ -196,66 +196,66 @@ apple => [count=10, txid=2]
<h2 id="state-apis">State APIs</h2>
<p>You've seen the intricacies of what it takes to achieve exactly-once semantics. The nice thing about Trident is that it internalizes all the fault-tolerance logic within the State – as a user you don't have to deal with comparing txids, storing multiple values in the database, or anything like that. You can write code like this:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TridentTopology</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TridentTopology</span><span class="o">();</span>
<span class="n">TridentState</span> <span class="n">wordCounts</span> <span class="o">=</span>
- <span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"spout1"</span><span class="o">,</span> <span class="n">spout</span><span class="o">)</span>
- <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"sentence"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Split</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">persistentAggregate</span><span class="o">(</span><span class="n">MemcachedState</span><span class="o">.</span><span class="na">opaque</span><span class="o">(</span><span class="n">serverLocations</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
+ <span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"spout1"</span><span class="o">,</span> <span class="n">spout</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"sentence"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Split</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">persistentAggregate</span><span class="o">(</span><span class="n">MemcachedState</span><span class="o">.</span><span class="na">opaque</span><span class="o">(</span><span class="n">serverLocations</span><span class="o">),</span> <span class="k">new</span> <span class="n">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
<span class="o">.</span><span class="na">parallelismHint</span><span class="o">(</span><span class="mi">6</span><span class="o">);</span>
</code></pre></div>
<p>All the logic necessary to manage opaque transactional state logic is internalized in the MemcachedState.opaque call. Additionally, updates are automatically batched to minimize roundtrips to the database.</p>
<p>The base State interface just has two methods:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">State</span> <span class="o">{</span>
- <span class="kt">void</span> <span class="nf">beginCommit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">);</span> <span class="c1">// can be null for things like partitionPersist occurring off a DRPC stream</span>
- <span class="kt">void</span> <span class="nf">commit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">beginCommit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">);</span> <span class="c1">// can be null for things like partitionPersist occurring off a DRPC stream</span>
+ <span class="kt">void</span> <span class="n">commit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>You're told when a state update is beginning, when a state update is ending, and you're given the txid in each case. Trident assumes nothing about how your state works, what kind of methods there are to update it, and what kind of methods there are to read from it.</p>
<p>Suppose you have a home-grown database that contains user location information and you want to be able to access it from Trident. Your State implementation would have methods for getting and setting user information:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">LocationDB</span> <span class="kd">implements</span> <span class="n">State</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">beginCommit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">beginCommit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">)</span> <span class="o">{</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">commit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">commit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">)</span> <span class="o">{</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">setLocation</span><span class="o">(</span><span class="kt">long</span> <span class="n">userId</span><span class="o">,</span> <span class="n">String</span> <span class="n">location</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">setLocation</span><span class="o">(</span><span class="kt">long</span> <span class="n">userId</span><span class="o">,</span> <span class="n">String</span> <span class="n">location</span><span class="o">)</span> <span class="o">{</span>
<span class="c1">// code to access database and set location</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="n">String</span> <span class="nf">getLocation</span><span class="o">(</span><span class="kt">long</span> <span class="n">userId</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">String</span> <span class="n">getLocation</span><span class="o">(</span><span class="kt">long</span> <span class="n">userId</span><span class="o">)</span> <span class="o">{</span>
<span class="c1">// code to get location from database</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
<p>You then provide Trident a StateFactory that can create instances of your State object within Trident tasks. The StateFactory for your LocationDB might look something like this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">LocationDBFactory</span> <span class="kd">implements</span> <span class="n">StateFactory</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="n">State</span> <span class="nf">makeState</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="kt">int</span> <span class="n">partitionIndex</span><span class="o">,</span> <span class="kt">int</span> <span class="n">numPartitions</span><span class="o">)</span> <span class="o">{</span>
- <span class="k">return</span> <span class="k">new</span> <span class="nf">LocationDB</span><span class="o">();</span>
+ <span class="kd">public</span> <span class="n">State</span> <span class="n">makeState</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="kt">int</span> <span class="n">partitionIndex</span><span class="o">,</span> <span class="kt">int</span> <span class="n">numPartitions</span><span class="o">)</span> <span class="o">{</span>
+ <span class="k">return</span> <span class="k">new</span> <span class="n">LocationDB</span><span class="o">();</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
<p>Trident provides the QueryFunction interface for writing Trident operations that query a source of state, and the StateUpdater interface for writing Trident operations that update a source of state. For example, let's write an operation "QueryLocation" that queries the LocationDB for the locations of users. Let's start off with how you would use it in a topology. Let's say this topology consumes an input stream of userids:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TridentTopology</span><span class="o">();</span>
-<span class="n">TridentState</span> <span class="n">locations</span> <span class="o">=</span> <span class="n">topology</span><span class="o">.</span><span class="na">newStaticState</span><span class="o">(</span><span class="k">new</span> <span class="nf">LocationDBFactory</span><span class="o">());</span>
-<span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"myspout"</span><span class="o">,</span> <span class="n">spout</span><span class="o">)</span>
- <span class="o">.</span><span class="na">stateQuery</span><span class="o">(</span><span class="n">locations</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"userid"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">QueryLocation</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"location"</span><span class="o">))</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TridentTopology</span><span class="o">();</span>
+<span class="n">TridentState</span> <span class="n">locations</span> <span class="o">=</span> <span class="n">topology</span><span class="o">.</span><span class="na">newStaticState</span><span class="o">(</span><span class="k">new</span> <span class="n">LocationDBFactory</span><span class="o">());</span>
+<span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"myspout"</span><span class="o">,</span> <span class="n">spout</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">stateQuery</span><span class="o">(</span><span class="n">locations</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"userid"</span><span class="o">),</span> <span class="k">new</span> <span class="n">QueryLocation</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"location"</span><span class="o">))</span>
</code></pre></div>
<p>Now let's take a look at what the implementation of QueryLocation would look like:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">QueryLocation</span> <span class="kd">extends</span> <span class="n">BaseQueryFunction</span><span class="o"><</span><span class="n">LocationDB</span><span class="o">,</span> <span class="n">String</span><span class="o">></span> <span class="o">{</span>
- <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="nf">batchRetrieve</span><span class="o">(</span><span class="n">LocationDB</span> <span class="n">state</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">TridentTuple</span><span class="o">></span> <span class="n">inputs</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">ret</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">ArrayList</span><span class="o">();</span>
+ <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">batchRetrieve</span><span class="o">(</span><span class="n">LocationDB</span> <span class="n">state</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">TridentTuple</span><span class="o">></span> <span class="n">inputs</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">ret</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o">();</span>
<span class="k">for</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="nl">input:</span> <span class="n">inputs</span><span class="o">)</span> <span class="o">{</span>
<span class="n">ret</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="n">state</span><span class="o">.</span><span class="na">getLocation</span><span class="o">(</span><span class="n">input</span><span class="o">.</span><span class="na">getLong</span><span class="o">(</span><span class="mi">0</span><span class="o">)));</span>
<span class="o">}</span>
<span class="k">return</span> <span class="n">ret</span><span class="o">;</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">String</span> <span class="n">location</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">location</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">String</span> <span class="n">location</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">location</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
@@ -263,24 +263,24 @@ apple => [count=10, txid=2]
<p>You can see that this code doesn't take advantage of the batching that Trident does, since it just queries the LocationDB one at a time. So a better way to write the LocationDB would be like this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">LocationDB</span> <span class="kd">implements</span> <span class="n">State</span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">beginCommit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">beginCommit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">)</span> <span class="o">{</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">commit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">commit</span><span class="o">(</span><span class="n">Long</span> <span class="n">txid</span><span class="o">)</span> <span class="o">{</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">setLocationsBulk</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="n">userIds</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">locations</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">setLocationsBulk</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="n">userIds</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">locations</span><span class="o">)</span> <span class="o">{</span>
<span class="c1">// set locations in bulk</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="nf">bulkGetLocations</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="n">userIds</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">bulkGetLocations</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="n">userIds</span><span class="o">)</span> <span class="o">{</span>
<span class="c1">// get locations in bulk</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
<p>Then, you can write the QueryLocation function like this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">QueryLocation</span> <span class="kd">extends</span> <span class="n">BaseQueryFunction</span><span class="o"><</span><span class="n">LocationDB</span><span class="o">,</span> <span class="n">String</span><span class="o">></span> <span class="o">{</span>
- <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="nf">batchRetrieve</span><span class="o">(</span><span class="n">LocationDB</span> <span class="n">state</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">TridentTuple</span><span class="o">></span> <span class="n">inputs</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">batchRetrieve</span><span class="o">(</span><span class="n">LocationDB</span> <span class="n">state</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">TridentTuple</span><span class="o">></span> <span class="n">inputs</span><span class="o">)</span> <span class="o">{</span>
<span class="n">List</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="n">userIds</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o"><</span><span class="n">Long</span><span class="o">>();</span>
<span class="k">for</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="nl">input:</span> <span class="n">inputs</span><span class="o">)</span> <span class="o">{</span>
<span class="n">userIds</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="n">input</span><span class="o">.</span><span class="na">getLong</span><span class="o">(</span><span class="mi">0</span><span class="o">));</span>
@@ -288,8 +288,8 @@ apple => [count=10, txid=2]
<span class="k">return</span> <span class="n">state</span><span class="o">.</span><span class="na">bulkGetLocations</span><span class="o">(</span><span class="n">userIds</span><span class="o">);</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">String</span> <span class="n">location</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">location</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">String</span> <span class="n">location</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">location</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
@@ -297,7 +297,7 @@ apple => [count=10, txid=2]
<p>To update state, you make use of the StateUpdater interface. Here's a StateUpdater that updates a LocationDB with new location information:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">LocationUpdater</span> <span class="kd">extends</span> <span class="n">BaseStateUpdater</span><span class="o"><</span><span class="n">LocationDB</span><span class="o">></span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">updateState</span><span class="o">(</span><span class="n">LocationDB</span> <span class="n">state</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">TridentTuple</span><span class="o">></span> <span class="n">tuples</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">updateState</span><span class="o">(</span><span class="n">LocationDB</span> <span class="n">state</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">TridentTuple</span><span class="o">></span> <span class="n">tuples</span><span class="o">,</span> <span class="n">TridentCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span>
<span class="n">List</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="n">ids</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o"><</span><span class="n">Long</span><span class="o">>();</span>
<span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">locations</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o"><</span><span class="n">String</span><span class="o">>();</span>
<span class="k">for</span><span class="o">(</span><span class="n">TridentTuple</span> <span class="nl">t:</span> <span class="n">tuples</span><span class="o">)</span> <span class="o">{</span>
@@ -309,10 +309,10 @@ apple => [count=10, txid=2]
<span class="o">}</span>
</code></pre></div>
<p>Here's how you would use this operation in a Trident topology:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TridentTopology</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TridentTopology</span><span class="o">();</span>
<span class="n">TridentState</span> <span class="n">locations</span> <span class="o">=</span>
- <span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"locations"</span><span class="o">,</span> <span class="n">locationsSpout</span><span class="o">)</span>
- <span class="o">.</span><span class="na">partitionPersist</span><span class="o">(</span><span class="k">new</span> <span class="nf">LocationDBFactory</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"userid"</span><span class="o">,</span> <span class="s">"location"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">LocationUpdater</span><span class="o">())</span>
+ <span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"locations"</span><span class="o">,</span> <span class="n">locationsSpout</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">partitionPersist</span><span class="o">(</span><span class="k">new</span> <span class="n">LocationDBFactory</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"userid"</span><span class="o">,</span> <span class="s">"location"</span><span class="o">),</span> <span class="k">new</span> <span class="n">LocationUpdater</span><span class="o">())</span>
</code></pre></div>
<p>The partitionPersist operation updates a source of state. The StateUpdater receives the State and a batch of tuples with updates to that State. This code just grabs the userids and locations from the input tuples and does a bulk set into the State. </p>
@@ -323,25 +323,25 @@ apple => [count=10, txid=2]
<h2 id="persistentaggregate">persistentAggregate</h2>
<p>Trident has another method for updating States called persistentAggregate. You've seen this used in the streaming word count example, shown again below:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TridentTopology</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TridentTopology</span><span class="o">();</span>
<span class="n">TridentState</span> <span class="n">wordCounts</span> <span class="o">=</span>
- <span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"spout1"</span><span class="o">,</span> <span class="n">spout</span><span class="o">)</span>
- <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"sentence"</span><span class="o">),</span> <span class="k">new</span> <span class="nf">Split</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
- <span class="o">.</span><span class="na">persistentAggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">MemoryMapState</span><span class="o">.</span><span class="na">Factory</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
+ <span class="n">topology</span><span class="o">.</span><span class="na">newStream</span><span class="o">(</span><span class="s">"spout1"</span><span class="o">,</span> <span class="n">spout</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">each</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"sentence"</span><span class="o">),</span> <span class="k">new</span> <span class="n">Split</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">))</span>
+ <span class="o">.</span><span class="na">persistentAggregate</span><span class="o">(</span><span class="k">new</span> <span class="n">MemoryMapState</span><span class="o">.</span><span class="na">Factory</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Count</span><span class="o">(),</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"count"</span><span class="o">))</span>
</code></pre></div>
<p>persistentAggregate is an additional abstraction built on top of partitionPersist that knows how to take a Trident aggregator and use it to apply updates to the source of state. In this case, since this is a grouped stream, Trident expects the state you provide to implement the "MapState" interface. The grouping fields will be the keys in the state, and the aggregation result will be the values in the state. The "MapState" interface looks like this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">MapState</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="kd">extends</span> <span class="n">State</span> <span class="o">{</span>
- <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="nf">multiGet</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">);</span>
- <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="nf">multiUpdate</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">ValueUpdater</span><span class="o">></span> <span class="n">updaters</span><span class="o">);</span>
- <span class="kt">void</span> <span class="nf">multiPut</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="n">vals</span><span class="o">);</span>
+ <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="n">multiGet</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">);</span>
+ <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="n">multiUpdate</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">ValueUpdater</span><span class="o">></span> <span class="n">updaters</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">multiPut</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="n">vals</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>When you do aggregations on non-grouped streams (a global aggregation), Trident expects your State object to implement the "Snapshottable" interface:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">Snapshottable</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="kd">extends</span> <span class="n">State</span> <span class="o">{</span>
- <span class="n">T</span> <span class="nf">get</span><span class="o">();</span>
- <span class="n">T</span> <span class="nf">update</span><span class="o">(</span><span class="n">ValueUpdater</span> <span class="n">updater</span><span class="o">);</span>
- <span class="kt">void</span> <span class="nf">set</span><span class="o">(</span><span class="n">T</span> <span class="n">o</span><span class="o">);</span>
+ <span class="n">T</span> <span class="n">get</span><span class="o">();</span>
+ <span class="n">T</span> <span class="n">update</span><span class="o">(</span><span class="n">ValueUpdater</span> <span class="n">updater</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">set</span><span class="o">(</span><span class="n">T</span> <span class="n">o</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p><a href="https://github.com/apache/storm/blob/master/storm-core/src/jvm/storm/trident/testing/MemoryMapState.java">MemoryMapState</a> and <a href="https://github.com/nathanmarz/trident-memcached/blob/master/src/jvm/trident/memcached/MemcachedState.java">MemcachedState</a> each implement both of these interfaces.</p>
@@ -350,8 +350,8 @@ apple => [count=10, txid=2]
<p>Trident makes it easy to implement MapState's, doing almost all the work for you. The OpaqueMap, TransactionalMap, and NonTransactionalMap classes implement all the logic for doing the respective fault-tolerance logic. You simply provide these classes with an IBackingMap implementation that knows how to do multiGets and multiPuts of the respective key/values. IBackingMap looks like this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">IBackingMap</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="o">{</span>
- <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="nf">multiGet</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">);</span>
- <span class="kt">void</span> <span class="nf">multiPut</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="n">vals</span><span class="o">);</span>
+ <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="n">multiGet</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">);</span>
+ <span class="kt">void</span> <span class="n">multiPut</span><span class="o">(</span><span class="n">List</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">keys</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="n">vals</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>OpaqueMap's will call multiPut with <a href="https://github.com/apache/storm/blob/master/storm-core/src/jvm/storm/trident/state/OpaqueValue.java">OpaqueValue</a>'s for the vals, TransactionalMap's will give <a href="https://github.com/apache/storm/blob/master/storm-core/src/jvm/storm/trident/state/TransactionalValue.java">TransactionalValue</a>'s for the vals, and NonTransactionalMaps will just pass the objects from the topology through.</p>
[04/10] storm git commit: STORM-1366. Add documentation for StormSQL
into the website.
Posted by wh...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/flux.html
----------------------------------------------------------------------
diff --git a/_site/documentation/flux.html b/_site/documentation/flux.html
index 8674858..01126ff 100644
--- a/_site/documentation/flux.html
+++ b/_site/documentation/flux.html
@@ -114,12 +114,13 @@ order to change configuration.</p>
deveoper-intensive.</p>
<p>Have you ever found yourself repeating this pattern?:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span>
- <span class="c1">// logic to determine if we're running locally or not...</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java">
+<span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="p">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span>
+ <span class="c1">// logic to determine if we're running locally or not...</span>
<span class="c1">// create necessary config options...</span>
<span class="kt">boolean</span> <span class="n">runLocal</span> <span class="o">=</span> <span class="n">shouldRunLocal</span><span class="o">();</span>
<span class="k">if</span><span class="o">(</span><span class="n">runLocal</span><span class="o">){</span>
- <span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">LocalCluster</span><span class="o">();</span>
+ <span class="n">LocalCluster</span> <span class="n">cluster</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LocalCluster</span><span class="o">();</span>
<span class="n">cluster</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="n">name</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">topology</span><span class="o">);</span>
<span class="o">}</span> <span class="k">else</span> <span class="o">{</span>
<span class="n">StormSubmitter</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="n">name</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">topology</span><span class="o">);</span>
@@ -167,19 +168,19 @@ on your system:</p>
<li>Node.js 0.10.x or later</li>
</ul>
-<h4 id="building-with-unit-tests-enabled:">Building with unit tests enabled:</h4>
-<div class="highlight"><pre><code class="language-text" data-lang="text">mvn clean install
+<h4 id="building-with-unit-tests-enabled">Building with unit tests enabled:</h4>
+<div class="highlight"><pre><code class="language-" data-lang="">mvn clean install
</code></pre></div>
-<h4 id="building-with-unit-tests-disabled:">Building with unit tests disabled:</h4>
+<h4 id="building-with-unit-tests-disabled">Building with unit tests disabled:</h4>
<p>If you would like to build Flux without installing Python or Node.js you can simply skip the unit tests:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">mvn clean install -DskipTests=true
+<div class="highlight"><pre><code class="language-" data-lang="">mvn clean install -DskipTests=true
</code></pre></div>
<p>Note that if you plan on using Flux to deploy topologies to a remote cluster, you will still need to have Python
installed since it is required by Apache Storm.</p>
-<h4 id="building-with-integration-tests-enabled:">Building with integration tests enabled:</h4>
-<div class="highlight"><pre><code class="language-text" data-lang="text">mvn clean install -DskipIntegration=false
+<h4 id="building-with-integration-tests-enabled">Building with integration tests enabled:</h4>
+<div class="highlight"><pre><code class="language-" data-lang="">mvn clean install -DskipIntegration=false
</code></pre></div>
<h3 id="packaging-with-maven">Packaging with Maven</h3>
@@ -232,9 +233,9 @@ recommended).</p>
<span class="nt"><configuration></span>
<span class="nt"><transformers></span>
<span class="nt"><transformer</span>
- <span class="na">implementation=</span><span class="s">"org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"</span><span class="nt">/></span>
+ <span class="na">implementation=</span><span class="s">"org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"</span><span class="nt">/></span>
<span class="nt"><transformer</span>
- <span class="na">implementation=</span><span class="s">"org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"</span><span class="nt">></span>
+ <span class="na">implementation=</span><span class="s">"org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"</span><span class="nt">></span>
<span class="nt"><mainClass></span>org.apache.storm.flux.Flux<span class="nt"></mainClass></span>
<span class="nt"></transformer></span>
<span class="nt"></transformers></span>
@@ -251,9 +252,10 @@ recommended).</p>
or remotely using the <code>storm jar</code> command. For example, if your fat jar is named <code>myTopology-0.1.0-SNAPSHOT.jar</code> you
could run it locally with the command:</p>
<div class="highlight"><pre><code class="language-bash" data-lang="bash">storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --local my_config.yaml
+
</code></pre></div>
<h3 id="command-line-options">Command line options</h3>
-<div class="highlight"><pre><code class="language-text" data-lang="text">usage: storm jar <my_topology_uber_jar.jar> org.apache.storm.flux.Flux
+<div class="highlight"><pre><code class="language-" data-lang="">usage: storm jar <my_topology_uber_jar.jar> org.apache.storm.flux.Flux
[options] <topology-config.yaml>
-d,--dry-run Do not run or deploy the topology. Just
build, validate, and print information about
@@ -290,7 +292,7 @@ example command will run Flux and override the <code>nimus.host</code> configura
<div class="highlight"><pre><code class="language-bash" data-lang="bash">storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --remote my_config.yaml -c nimbus.host<span class="o">=</span>localhost
</code></pre></div>
<h3 id="sample-output">Sample output</h3>
-<div class="highlight"><pre><code class="language-text" data-lang="text">███████╗██╗ ██╗ ██╗██╗ ██╗
+<div class="highlight"><pre><code class="language-" data-lang="">███████╗██╗ ██╗ ██╗██╗ ██╗
██╔════╝██║ ██║ ██║╚██╗██╔╝
█████╗ ██║ ██║ ██║ ╚███╔╝
██╔══╝ ██║ ██║ ██║ ██╔██╗
@@ -313,7 +315,7 @@ sentence-spout --SHUFFLE--> splitsentence
splitsentence --FIELDS--> count
count --SHUFFLE--> log
--------------------------------------
-Submitting topology: 'shell-topology' to remote cluster...
+Submitting topology: 'shell-topology' to remote cluster...
</code></pre></div>
<h2 id="yaml-configuration">YAML Configuration</h2>
@@ -338,41 +340,43 @@ definition consists of the following:</p>
</ol>
<p>For example, here is a simple definition of a wordcount topology using the YAML DSL:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"yaml-topology"</span>
-<span class="l-Scalar-Plain">config</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">topology.workers</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">yaml-topology"</span>
+<span class="s">config</span><span class="pi">:</span>
+ <span class="s">topology.workers</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1"># spout definitions</span>
-<span class="l-Scalar-Plain">spouts</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"spout-1"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"backtype.storm.testing.TestWordSpout"</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+<span class="s">spouts</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">spout-1"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">backtype.storm.testing.TestWordSpout"</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1"># bolt definitions</span>
-<span class="l-Scalar-Plain">bolts</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"bolt-1"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"backtype.storm.testing.TestWordCounter"</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"bolt-2"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.wrappers.bolts.LogInfoBolt"</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+<span class="s">bolts</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-1"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">backtype.storm.testing.TestWordCounter"</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-2"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.wrappers.bolts.LogInfoBolt"</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1">#stream definitions</span>
-<span class="l-Scalar-Plain">streams</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"spout-1</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">bolt-1"</span> <span class="c1"># name isn't used (placeholder for logging, UI, etc.)</span>
- <span class="l-Scalar-Plain">from</span><span class="p-Indicator">:</span> <span class="s">"spout-1"</span>
- <span class="l-Scalar-Plain">to</span><span class="p-Indicator">:</span> <span class="s">"bolt-1"</span>
- <span class="l-Scalar-Plain">grouping</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">FIELDS</span>
- <span class="l-Scalar-Plain">args</span><span class="p-Indicator">:</span> <span class="p-Indicator">[</span><span class="s">"word"</span><span class="p-Indicator">]</span>
-
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"bolt-1</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">bolt2"</span>
- <span class="l-Scalar-Plain">from</span><span class="p-Indicator">:</span> <span class="s">"bolt-1"</span>
- <span class="l-Scalar-Plain">to</span><span class="p-Indicator">:</span> <span class="s">"bolt-2"</span>
- <span class="l-Scalar-Plain">grouping</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">SHUFFLE</span>
-</code></pre></div>
-<h2 id="property-substitution/filtering">Property Substitution/Filtering</h2>
+<span class="s">streams</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">spout-1</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">bolt-1"</span> <span class="c1"># name isn't used (placeholder for logging, UI, etc.)</span>
+ <span class="s">from</span><span class="pi">:</span> <span class="s2">"</span><span class="s">spout-1"</span>
+ <span class="s">to</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-1"</span>
+ <span class="s">grouping</span><span class="pi">:</span>
+ <span class="s">type</span><span class="pi">:</span> <span class="s">FIELDS</span>
+ <span class="s">args</span><span class="pi">:</span> <span class="pi">[</span><span class="s2">"</span><span class="s">word"</span><span class="pi">]</span>
+
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-1</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">bolt2"</span>
+ <span class="s">from</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-1"</span>
+ <span class="s">to</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-2"</span>
+ <span class="s">grouping</span><span class="pi">:</span>
+ <span class="s">type</span><span class="pi">:</span> <span class="s">SHUFFLE</span>
+
+
+</code></pre></div>
+<h2 id="property-substitution-filtering">Property Substitution/Filtering</h2>
<p>It's common for developers to want to easily switch between configurations, for example switching deployment between
a development environment and a production environment. This can be accomplished by using separate YAML configuration
@@ -387,21 +391,21 @@ if you invoked flux like so:</p>
<div class="highlight"><pre><code class="language-bash" data-lang="bash">storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --local my_config.yaml --filter dev.properties
</code></pre></div>
<p>With the following <code>dev.properties</code> file:</p>
-<div class="highlight"><pre><code class="language-properties" data-lang="properties"><span class="na">kafka.zookeeper.hosts</span><span class="o">:</span> <span class="s">localhost:2181</span>
+<div class="highlight"><pre><code class="language-properties" data-lang="properties"><span class="py">kafka.zookeeper.hosts</span><span class="p">:</span> <span class="s">localhost:2181</span>
</code></pre></div>
<p>You would then be able to reference those properties by key in your <code>.yaml</code> file using <code>${}</code> syntax:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"zkHosts"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"storm.kafka.ZkHosts"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="s">"${kafka.zookeeper.hosts}"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">zkHosts"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">storm.kafka.ZkHosts"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">${kafka.zookeeper.hosts}"</span>
</code></pre></div>
<p>In this case, Flux would replace <code>${kafka.zookeeper.hosts}</code> with <code>localhost:2181</code> before parsing the YAML contents.</p>
-<h3 id="environment-variable-substitution/filtering">Environment Variable Substitution/Filtering</h3>
+<h3 id="environment-variable-substitution-filtering">Environment Variable Substitution/Filtering</h3>
<p>Flux also allows environment variable substitution. For example, if an environment variable named <code>ZK_HOSTS</code> if defined,
you can reference it in a Flux YAML file with the following syntax:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">${ENV-ZK_HOSTS}
+<div class="highlight"><pre><code class="language-" data-lang="">${ENV-ZK_HOSTS}
</code></pre></div>
<h2 id="components">Components</h2>
@@ -411,21 +415,21 @@ bolts. If you are familiar with the Spring framework, components are roughly ana
<p>Every component is identified, at a minimum, by a unique identifier (String) and a class name (String). For example,
the following will make an instance of the <code>storm.kafka.StringScheme</code> class available as a reference under the key
<code>"stringScheme"</code> . This assumes the <code>storm.kafka.StringScheme</code> has a default constructor.</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">components</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"stringScheme"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"storm.kafka.StringScheme"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">components</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">stringScheme"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">storm.kafka.StringScheme"</span>
</code></pre></div>
-<h3 id="contructor-arguments,-references,-properties-and-configuration-methods">Contructor Arguments, References, Properties and Configuration Methods</h3>
+<h3 id="contructor-arguments-references-properties-and-configuration-methods">Contructor Arguments, References, Properties and Configuration Methods</h3>
<h4 id="constructor-arguments">Constructor Arguments</h4>
<p>Arguments to a class constructor can be configured by adding a <code>contructorArgs</code> element to a components.
<code>constructorArgs</code> is a list of objects that will be passed to the class' constructor. The following example creates an
object by calling the constructor that takes a single string as an argument:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"zkHosts"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"storm.kafka.ZkHosts"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="s">"localhost:2181"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">zkHosts"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">storm.kafka.ZkHosts"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">localhost:2181"</span>
</code></pre></div>
<h4 id="references">References</h4>
@@ -434,14 +438,14 @@ reference an existing component, you specify the id of the component with the <c
<p>In the following example, a component with the id <code>"stringScheme"</code> is created, and later referenced, as a an argument
to another component's constructor:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">components</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"stringScheme"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"storm.kafka.StringScheme"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">components</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">stringScheme"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">storm.kafka.StringScheme"</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"stringMultiScheme"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"backtype.storm.spout.SchemeAsMultiScheme"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">ref</span><span class="p-Indicator">:</span> <span class="s">"stringScheme"</span> <span class="c1"># component with id "stringScheme" must be declared above.</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">stringMultiScheme"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">backtype.storm.spout.SchemeAsMultiScheme"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">ref</span><span class="pi">:</span> <span class="s2">"</span><span class="s">stringScheme"</span> <span class="c1"># component with id "stringScheme" must be declared above.</span>
</code></pre></div>
<p><strong>N.B.:</strong> References can only be used after (below) the object they point to has been declared.</p>
@@ -449,22 +453,22 @@ to another component's constructor:</p>
<p>In addition to calling constructors with different arguments, Flux also allows you to configure components using
JavaBean-like setter methods and fields declared as <code>public</code>:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"spoutConfig"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"storm.kafka.SpoutConfig"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">spoutConfig"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">storm.kafka.SpoutConfig"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
<span class="c1"># brokerHosts</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">ref</span><span class="p-Indicator">:</span> <span class="s">"zkHosts"</span>
+ <span class="pi">-</span> <span class="s">ref</span><span class="pi">:</span> <span class="s2">"</span><span class="s">zkHosts"</span>
<span class="c1"># topic</span>
- <span class="p-Indicator">-</span> <span class="s">"myKafkaTopic"</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">myKafkaTopic"</span>
<span class="c1"># zkRoot</span>
- <span class="p-Indicator">-</span> <span class="s">"/kafkaSpout"</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">/kafkaSpout"</span>
<span class="c1"># id</span>
- <span class="p-Indicator">-</span> <span class="s">"myId"</span>
- <span class="l-Scalar-Plain">properties</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"forceFromStart"</span>
- <span class="l-Scalar-Plain">value</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">true</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"scheme"</span>
- <span class="l-Scalar-Plain">ref</span><span class="p-Indicator">:</span> <span class="s">"stringMultiScheme"</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">myId"</span>
+ <span class="s">properties</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">forceFromStart"</span>
+ <span class="s">value</span><span class="pi">:</span> <span class="s">true</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">scheme"</span>
+ <span class="s">ref</span><span class="pi">:</span> <span class="s2">"</span><span class="s">stringMultiScheme"</span>
</code></pre></div>
<p>In the example above, the <code>properties</code> declaration will cause Flux to look for a public method in the <code>SpoutConfig</code> with
the signature <code>setForceFromStart(boolean b)</code> and attempt to invoke it. If a setter method is not found, Flux will then
@@ -480,31 +484,31 @@ don't expose JavaBean methods or have constructors that can fully configure
that use the builder pattern for configuration/composition.</p>
<p>The following YAML example creates a bolt and configures it by calling several methods:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">bolts</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"bolt-1"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.test.TestBolt"</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
- <span class="l-Scalar-Plain">configMethods</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"withFoo"</span>
- <span class="l-Scalar-Plain">args</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="s">"foo"</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"withBar"</span>
- <span class="l-Scalar-Plain">args</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="s">"bar"</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"withFooBar"</span>
- <span class="l-Scalar-Plain">args</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="s">"foo"</span>
- <span class="p-Indicator">-</span> <span class="s">"bar"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">bolts</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-1"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.test.TestBolt"</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
+ <span class="s">configMethods</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">withFoo"</span>
+ <span class="s">args</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">foo"</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">withBar"</span>
+ <span class="s">args</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">bar"</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">withFooBar"</span>
+ <span class="s">args</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">foo"</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">bar"</span>
</code></pre></div>
<p>The signatures of the corresponding methods are as follows:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="kd">public</span> <span class="kt">void</span> <span class="nf">withFoo</span><span class="o">(</span><span class="n">String</span> <span class="n">foo</span><span class="o">);</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">withBar</span><span class="o">(</span><span class="n">String</span> <span class="n">bar</span><span class="o">);</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">withFooBar</span><span class="o">(</span><span class="n">String</span> <span class="n">foo</span><span class="o">,</span> <span class="n">String</span> <span class="n">bar</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="kd">public</span> <span class="kt">void</span> <span class="nf">withFoo</span><span class="p">(</span><span class="n">String</span> <span class="n">foo</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">withBar</span><span class="o">(</span><span class="n">String</span> <span class="n">bar</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">withFooBar</span><span class="o">(</span><span class="n">String</span> <span class="n">foo</span><span class="o">,</span> <span class="n">String</span> <span class="n">bar</span><span class="o">);</span>
</code></pre></div>
<p>Arguments passed to configuration methods work much the same way as constructor arguments, and support references as
well.</p>
-<h3 id="using-java-enums-in-contructor-arguments,-references,-properties-and-configuration-methods">Using Java <code>enum</code>s in Contructor Arguments, References, Properties and Configuration Methods</h3>
+<h3 id="using-java-enums-in-contructor-arguments-references-properties-and-configuration-methods">Using Java <code>enum</code>s in Contructor Arguments, References, Properties and Configuration Methods</h3>
<p>You can easily use Java <code>enum</code> values as arguments in a Flux YAML file, simply by referencing the name of the <code>enum</code>.</p>
@@ -514,27 +518,28 @@ well.</p>
<span class="o">}</span>
</code></pre></div>
<p>And the <code>org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy</code> class has the following constructor:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="nf">FileSizeRotationPolicy</span><span class="o">(</span><span class="kt">float</span> <span class="n">count</span><span class="o">,</span> <span class="n">Units</span> <span class="n">units</span><span class="o">)</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="nf">FileSizeRotationPolicy</span><span class="p">(</span><span class="kt">float</span> <span class="n">count</span><span class="o">,</span> <span class="n">Units</span> <span class="n">units</span><span class="o">)</span>
+
</code></pre></div>
<p>The following Flux <code>component</code> definition could be used to call the constructor:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"rotationPolicy"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">5.0</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">MB</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">rotationPolicy"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">5.0</span>
+ <span class="pi">-</span> <span class="s">MB</span>
</code></pre></div>
<p>The above definition is functionally equivalent to the following Java code:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="c1">// rotate files when they reach 5MB</span>
-<span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
+<span class="n">FileRotationPolicy</span> <span class="n">rotationPolicy</span> <span class="o">=</span> <span class="k">new</span> <span class="n">FileSizeRotationPolicy</span><span class="o">(</span><span class="mf">5.0f</span><span class="o">,</span> <span class="n">Units</span><span class="o">.</span><span class="na">MB</span><span class="o">);</span>
</code></pre></div>
<h2 id="topology-config">Topology Config</h2>
<p>The <code>config</code> section is simply a map of Storm topology configuration parameters that will be passed to the
<code>backtype.storm.StormSubmitter</code> as an instance of the <code>backtype.storm.Config</code> class:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">config</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">topology.workers</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">4</span>
- <span class="l-Scalar-Plain">topology.max.spout.pending</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1000</span>
- <span class="l-Scalar-Plain">topology.message.timeout.secs</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">30</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">config</span><span class="pi">:</span>
+ <span class="s">topology.workers</span><span class="pi">:</span> <span class="s">4</span>
+ <span class="s">topology.max.spout.pending</span><span class="pi">:</span> <span class="s">1000</span>
+ <span class="s">topology.message.timeout.secs</span><span class="pi">:</span> <span class="s">30</span>
</code></pre></div>
<h1 id="existing-topologies">Existing Topologies</h1>
@@ -544,22 +549,22 @@ classes.</p>
<p>The easiest way to use an existing topology class is to define
a <code>getTopology()</code> instance method with one of the following signatures:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="n">StormTopology</span> <span class="nf">getTopology</span><span class="o">(</span><span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Object</span><span class="o">></span> <span class="n">config</span><span class="o">)</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="n">StormTopology</span> <span class="nf">getTopology</span><span class="p">(</span><span class="n">Map</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Object</span><span class="o">></span> <span class="n">config</span><span class="o">)</span>
</code></pre></div>
<p>or:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="n">StormTopology</span> <span class="nf">getTopology</span><span class="o">(</span><span class="n">Config</span> <span class="n">config</span><span class="o">)</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="n">StormTopology</span> <span class="nf">getTopology</span><span class="p">(</span><span class="n">Config</span> <span class="n">config</span><span class="o">)</span>
</code></pre></div>
<p>You could then use the following YAML to configure your topology:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"existing-topology"</span>
-<span class="l-Scalar-Plain">topologySource</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.test.SimpleTopology"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">existing-topology"</span>
+<span class="s">topologySource</span><span class="pi">:</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.test.SimpleTopology"</span>
</code></pre></div>
<p>If the class you would like to use as a topology source has a different method name (i.e. not <code>getTopology</code>), you can
override it:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"existing-topology"</span>
-<span class="l-Scalar-Plain">topologySource</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.test.SimpleTopology"</span>
- <span class="l-Scalar-Plain">methodName</span><span class="p-Indicator">:</span> <span class="s">"getTopologyWithDifferentMethodName"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">existing-topology"</span>
+<span class="s">topologySource</span><span class="pi">:</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.test.SimpleTopology"</span>
+ <span class="s">methodName</span><span class="pi">:</span> <span class="s2">"</span><span class="s">getTopologyWithDifferentMethodName"</span>
</code></pre></div>
<p><strong>N.B.:</strong> The specified method must accept a single argument of type <code>java.util.Map<String, Object></code> or
<code>backtype.storm.Config</code>, and return a <code>backtype.storm.generated.StormTopology</code> object.</p>
@@ -576,91 +581,92 @@ component when the topology is deployed.</p>
well.</p>
<p>Shell spout example:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">spouts</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"sentence-spout"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.spouts.GenericShellSpout"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">spouts</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">sentence-spout"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.spouts.GenericShellSpout"</span>
<span class="c1"># shell spout constructor takes 2 arguments: String[], String[]</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
<span class="c1"># command line</span>
- <span class="p-Indicator">-</span> <span class="p-Indicator">[</span><span class="s">"node"</span><span class="p-Indicator">,</span> <span class="s">"randomsentence.js"</span><span class="p-Indicator">]</span>
+ <span class="pi">-</span> <span class="pi">[</span><span class="s2">"</span><span class="s">node"</span><span class="pi">,</span> <span class="s2">"</span><span class="s">randomsentence.js"</span><span class="pi">]</span>
<span class="c1"># output fields</span>
- <span class="p-Indicator">-</span> <span class="p-Indicator">[</span><span class="s">"word"</span><span class="p-Indicator">]</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+ <span class="pi">-</span> <span class="pi">[</span><span class="s2">"</span><span class="s">word"</span><span class="pi">]</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
</code></pre></div>
<p>Kafka spout example:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">components</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"stringScheme"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"storm.kafka.StringScheme"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">components</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">stringScheme"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">storm.kafka.StringScheme"</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"stringMultiScheme"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"backtype.storm.spout.SchemeAsMultiScheme"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">ref</span><span class="p-Indicator">:</span> <span class="s">"stringScheme"</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">stringMultiScheme"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">backtype.storm.spout.SchemeAsMultiScheme"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">ref</span><span class="pi">:</span> <span class="s2">"</span><span class="s">stringScheme"</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"zkHosts"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"storm.kafka.ZkHosts"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="s">"localhost:2181"</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">zkHosts"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">storm.kafka.ZkHosts"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">localhost:2181"</span>
<span class="c1"># Alternative kafka config</span>
-<span class="c1"># - id: "kafkaConfig"</span>
-<span class="c1"># className: "storm.kafka.KafkaConfig"</span>
+<span class="c1"># - id: "kafkaConfig"</span>
+<span class="c1"># className: "storm.kafka.KafkaConfig"</span>
<span class="c1"># constructorArgs:</span>
<span class="c1"># # brokerHosts</span>
-<span class="c1"># - ref: "zkHosts"</span>
+<span class="c1"># - ref: "zkHosts"</span>
<span class="c1"># # topic</span>
-<span class="c1"># - "myKafkaTopic"</span>
+<span class="c1"># - "myKafkaTopic"</span>
<span class="c1"># # clientId (optional)</span>
-<span class="c1"># - "myKafkaClientId"</span>
+<span class="c1"># - "myKafkaClientId"</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"spoutConfig"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"storm.kafka.SpoutConfig"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">spoutConfig"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">storm.kafka.SpoutConfig"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
<span class="c1"># brokerHosts</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">ref</span><span class="p-Indicator">:</span> <span class="s">"zkHosts"</span>
+ <span class="pi">-</span> <span class="s">ref</span><span class="pi">:</span> <span class="s2">"</span><span class="s">zkHosts"</span>
<span class="c1"># topic</span>
- <span class="p-Indicator">-</span> <span class="s">"myKafkaTopic"</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">myKafkaTopic"</span>
<span class="c1"># zkRoot</span>
- <span class="p-Indicator">-</span> <span class="s">"/kafkaSpout"</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">/kafkaSpout"</span>
<span class="c1"># id</span>
- <span class="p-Indicator">-</span> <span class="s">"myId"</span>
- <span class="l-Scalar-Plain">properties</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"forceFromStart"</span>
- <span class="l-Scalar-Plain">value</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">true</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"scheme"</span>
- <span class="l-Scalar-Plain">ref</span><span class="p-Indicator">:</span> <span class="s">"stringMultiScheme"</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">myId"</span>
+ <span class="s">properties</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">forceFromStart"</span>
+ <span class="s">value</span><span class="pi">:</span> <span class="s">true</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">scheme"</span>
+ <span class="s">ref</span><span class="pi">:</span> <span class="s2">"</span><span class="s">stringMultiScheme"</span>
-<span class="l-Scalar-Plain">config</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">topology.workers</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+<span class="s">config</span><span class="pi">:</span>
+ <span class="s">topology.workers</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1"># spout definitions</span>
-<span class="l-Scalar-Plain">spouts</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"kafka-spout"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"storm.kafka.KafkaSpout"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">ref</span><span class="p-Indicator">:</span> <span class="s">"spoutConfig"</span>
+<span class="s">spouts</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">kafka-spout"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">storm.kafka.KafkaSpout"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">ref</span><span class="pi">:</span> <span class="s2">"</span><span class="s">spoutConfig"</span>
+
</code></pre></div>
<p>Bolt Examples:</p>
<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="c1"># bolt definitions</span>
-<span class="l-Scalar-Plain">bolts</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"splitsentence"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.bolts.GenericShellBolt"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
+<span class="s">bolts</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">splitsentence"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.bolts.GenericShellBolt"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
<span class="c1"># command line</span>
- <span class="p-Indicator">-</span> <span class="p-Indicator">[</span><span class="s">"python"</span><span class="p-Indicator">,</span> <span class="s">"splitsentence.py"</span><span class="p-Indicator">]</span>
+ <span class="pi">-</span> <span class="pi">[</span><span class="s2">"</span><span class="s">python"</span><span class="pi">,</span> <span class="s2">"</span><span class="s">splitsentence.py"</span><span class="pi">]</span>
<span class="c1"># output fields</span>
- <span class="p-Indicator">-</span> <span class="p-Indicator">[</span><span class="s">"word"</span><span class="p-Indicator">]</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+ <span class="pi">-</span> <span class="pi">[</span><span class="s2">"</span><span class="s">word"</span><span class="pi">]</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1"># ...</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"log"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.wrappers.bolts.LogInfoBolt"</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">log"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.wrappers.bolts.LogInfoBolt"</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1"># ...</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"count"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"backtype.storm.testing.TestWordCounter"</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">count"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">backtype.storm.testing.TestWordCounter"</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1"># ...</span>
</code></pre></div>
<h2 id="streams-and-stream-groupings">Streams and Stream Groupings</h2>
@@ -689,31 +695,31 @@ a topology, with an associated Grouping definition.</p>
<p><strong><code>customClass</code></strong> For the <code>CUSTOM</code> grouping, a definition of custom grouping class instance</p>
<p>The <code>streams</code> definition example below sets up a topology with the following wiring:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"> kafka-spout --> splitsentence --> count --> log
+<div class="highlight"><pre><code class="language-" data-lang=""> kafka-spout --> splitsentence --> count --> log
</code></pre></div><div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="c1">#stream definitions</span>
<span class="c1"># stream definitions define connections between spouts and bolts.</span>
<span class="c1"># note that such connections can be cyclical</span>
<span class="c1"># custom stream groupings are also supported</span>
-<span class="l-Scalar-Plain">streams</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"kafka</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">split"</span> <span class="c1"># name isn't used (placeholder for logging, UI, etc.)</span>
- <span class="l-Scalar-Plain">from</span><span class="p-Indicator">:</span> <span class="s">"kafka-spout"</span>
- <span class="l-Scalar-Plain">to</span><span class="p-Indicator">:</span> <span class="s">"splitsentence"</span>
- <span class="l-Scalar-Plain">grouping</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">SHUFFLE</span>
-
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"split</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">count"</span>
- <span class="l-Scalar-Plain">from</span><span class="p-Indicator">:</span> <span class="s">"splitsentence"</span>
- <span class="l-Scalar-Plain">to</span><span class="p-Indicator">:</span> <span class="s">"count"</span>
- <span class="l-Scalar-Plain">grouping</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">FIELDS</span>
- <span class="l-Scalar-Plain">args</span><span class="p-Indicator">:</span> <span class="p-Indicator">[</span><span class="s">"word"</span><span class="p-Indicator">]</span>
-
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"count</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">log"</span>
- <span class="l-Scalar-Plain">from</span><span class="p-Indicator">:</span> <span class="s">"count"</span>
- <span class="l-Scalar-Plain">to</span><span class="p-Indicator">:</span> <span class="s">"log"</span>
- <span class="l-Scalar-Plain">grouping</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">SHUFFLE</span>
+<span class="s">streams</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">kafka</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">split"</span> <span class="c1"># name isn't used (placeholder for logging, UI, etc.)</span>
+ <span class="s">from</span><span class="pi">:</span> <span class="s2">"</span><span class="s">kafka-spout"</span>
+ <span class="s">to</span><span class="pi">:</span> <span class="s2">"</span><span class="s">splitsentence"</span>
+ <span class="s">grouping</span><span class="pi">:</span>
+ <span class="s">type</span><span class="pi">:</span> <span class="s">SHUFFLE</span>
+
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">split</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">count"</span>
+ <span class="s">from</span><span class="pi">:</span> <span class="s2">"</span><span class="s">splitsentence"</span>
+ <span class="s">to</span><span class="pi">:</span> <span class="s2">"</span><span class="s">count"</span>
+ <span class="s">grouping</span><span class="pi">:</span>
+ <span class="s">type</span><span class="pi">:</span> <span class="s">FIELDS</span>
+ <span class="s">args</span><span class="pi">:</span> <span class="pi">[</span><span class="s2">"</span><span class="s">word"</span><span class="pi">]</span>
+
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">count</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">log"</span>
+ <span class="s">from</span><span class="pi">:</span> <span class="s2">"</span><span class="s">count"</span>
+ <span class="s">to</span><span class="pi">:</span> <span class="s2">"</span><span class="s">log"</span>
+ <span class="s">grouping</span><span class="pi">:</span>
+ <span class="s">type</span><span class="pi">:</span> <span class="s">SHUFFLE</span>
</code></pre></div>
<h3 id="custom-stream-groupings">Custom Stream Groupings</h3>
@@ -723,15 +729,15 @@ constructor arguments, references, and properties as well.</p>
<p>The example below creates a Stream with an instance of the <code>backtype.storm.testing.NGrouping</code> custom stream grouping
class.</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"bolt-1</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">bolt2"</span>
- <span class="l-Scalar-Plain">from</span><span class="p-Indicator">:</span> <span class="s">"bolt-1"</span>
- <span class="l-Scalar-Plain">to</span><span class="p-Indicator">:</span> <span class="s">"bolt-2"</span>
- <span class="l-Scalar-Plain">grouping</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">CUSTOM</span>
- <span class="l-Scalar-Plain">customClass</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"backtype.storm.testing.NGrouping"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">1</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"> <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-1</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">bolt2"</span>
+ <span class="s">from</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-1"</span>
+ <span class="s">to</span><span class="pi">:</span> <span class="s2">"</span><span class="s">bolt-2"</span>
+ <span class="s">grouping</span><span class="pi">:</span>
+ <span class="s">type</span><span class="pi">:</span> <span class="s">CUSTOM</span>
+ <span class="s">customClass</span><span class="pi">:</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">backtype.storm.testing.NGrouping"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">1</span>
</code></pre></div>
<h2 id="includes-and-overrides">Includes and Overrides</h2>
@@ -739,10 +745,10 @@ class.</p>
same file. Includes may be either files, or classpath resources.</p>
<p>Includes are specified as a list of maps:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">includes</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">resource</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">false</span>
- <span class="l-Scalar-Plain">file</span><span class="p-Indicator">:</span> <span class="s">"src/test/resources/configs/shell_test.yaml"</span>
- <span class="l-Scalar-Plain">override</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">false</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">includes</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">resource</span><span class="pi">:</span> <span class="s">false</span>
+ <span class="s">file</span><span class="pi">:</span> <span class="s2">"</span><span class="s">src/test/resources/configs/shell_test.yaml"</span>
+ <span class="s">override</span><span class="pi">:</span> <span class="s">false</span>
</code></pre></div>
<p>If the <code>resource</code> property is set to <code>true</code>, the include will be loaded as a classpath resource from the value of the
<code>file</code> attribute, otherwise it will be treated as a regular file.</p>
@@ -759,80 +765,80 @@ same file. Includes may be either files, or classpath resources.</p>
<p>Topology YAML config:</p>
<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="nn">---</span>
-<span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"shell-topology"</span>
-<span class="l-Scalar-Plain">config</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">topology.workers</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+<span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">shell-topology"</span>
+<span class="s">config</span><span class="pi">:</span>
+ <span class="s">topology.workers</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1"># spout definitions</span>
-<span class="l-Scalar-Plain">spouts</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"sentence-spout"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.spouts.GenericShellSpout"</span>
+<span class="s">spouts</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">sentence-spout"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.spouts.GenericShellSpout"</span>
<span class="c1"># shell spout constructor takes 2 arguments: String[], String[]</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
<span class="c1"># command line</span>
- <span class="p-Indicator">-</span> <span class="p-Indicator">[</span><span class="s">"node"</span><span class="p-Indicator">,</span> <span class="s">"randomsentence.js"</span><span class="p-Indicator">]</span>
+ <span class="pi">-</span> <span class="pi">[</span><span class="s2">"</span><span class="s">node"</span><span class="pi">,</span> <span class="s2">"</span><span class="s">randomsentence.js"</span><span class="pi">]</span>
<span class="c1"># output fields</span>
- <span class="p-Indicator">-</span> <span class="p-Indicator">[</span><span class="s">"word"</span><span class="p-Indicator">]</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+ <span class="pi">-</span> <span class="pi">[</span><span class="s2">"</span><span class="s">word"</span><span class="pi">]</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1"># bolt definitions</span>
-<span class="l-Scalar-Plain">bolts</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"splitsentence"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.bolts.GenericShellBolt"</span>
- <span class="l-Scalar-Plain">constructorArgs</span><span class="p-Indicator">:</span>
+<span class="s">bolts</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">splitsentence"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.bolts.GenericShellBolt"</span>
+ <span class="s">constructorArgs</span><span class="pi">:</span>
<span class="c1"># command line</span>
- <span class="p-Indicator">-</span> <span class="p-Indicator">[</span><span class="s">"python"</span><span class="p-Indicator">,</span> <span class="s">"splitsentence.py"</span><span class="p-Indicator">]</span>
+ <span class="pi">-</span> <span class="pi">[</span><span class="s2">"</span><span class="s">python"</span><span class="pi">,</span> <span class="s2">"</span><span class="s">splitsentence.py"</span><span class="pi">]</span>
<span class="c1"># output fields</span>
- <span class="p-Indicator">-</span> <span class="p-Indicator">[</span><span class="s">"word"</span><span class="p-Indicator">]</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+ <span class="pi">-</span> <span class="pi">[</span><span class="s2">"</span><span class="s">word"</span><span class="pi">]</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"log"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.wrappers.bolts.LogInfoBolt"</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">log"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.wrappers.bolts.LogInfoBolt"</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">id</span><span class="p-Indicator">:</span> <span class="s">"count"</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"backtype.storm.testing.TestWordCounter"</span>
- <span class="l-Scalar-Plain">parallelism</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+ <span class="pi">-</span> <span class="s">id</span><span class="pi">:</span> <span class="s2">"</span><span class="s">count"</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">backtype.storm.testing.TestWordCounter"</span>
+ <span class="s">parallelism</span><span class="pi">:</span> <span class="s">1</span>
<span class="c1">#stream definitions</span>
<span class="c1"># stream definitions define connections between spouts and bolts.</span>
<span class="c1"># note that such connections can be cyclical</span>
<span class="c1"># custom stream groupings are also supported</span>
-<span class="l-Scalar-Plain">streams</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"spout</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">split"</span> <span class="c1"># name isn't used (placeholder for logging, UI, etc.)</span>
- <span class="l-Scalar-Plain">from</span><span class="p-Indicator">:</span> <span class="s">"sentence-spout"</span>
- <span class="l-Scalar-Plain">to</span><span class="p-Indicator">:</span> <span class="s">"splitsentence"</span>
- <span class="l-Scalar-Plain">grouping</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">SHUFFLE</span>
+<span class="s">streams</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">spout</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">split"</span> <span class="c1"># name isn't used (placeholder for logging, UI, etc.)</span>
+ <span class="s">from</span><span class="pi">:</span> <span class="s2">"</span><span class="s">sentence-spout"</span>
+ <span class="s">to</span><span class="pi">:</span> <span class="s2">"</span><span class="s">splitsentence"</span>
+ <span class="s">grouping</span><span class="pi">:</span>
+ <span class="s">type</span><span class="pi">:</span> <span class="s">SHUFFLE</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"split</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">count"</span>
- <span class="l-Scalar-Plain">from</span><span class="p-Indicator">:</span> <span class="s">"splitsentence"</span>
- <span class="l-Scalar-Plain">to</span><span class="p-Indicator">:</span> <span class="s">"count"</span>
- <span class="l-Scalar-Plain">grouping</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">FIELDS</span>
- <span class="l-Scalar-Plain">args</span><span class="p-Indicator">:</span> <span class="p-Indicator">[</span><span class="s">"word"</span><span class="p-Indicator">]</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">split</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">count"</span>
+ <span class="s">from</span><span class="pi">:</span> <span class="s2">"</span><span class="s">splitsentence"</span>
+ <span class="s">to</span><span class="pi">:</span> <span class="s2">"</span><span class="s">count"</span>
+ <span class="s">grouping</span><span class="pi">:</span>
+ <span class="s">type</span><span class="pi">:</span> <span class="s">FIELDS</span>
+ <span class="s">args</span><span class="pi">:</span> <span class="pi">[</span><span class="s2">"</span><span class="s">word"</span><span class="pi">]</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"count</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">log"</span>
- <span class="l-Scalar-Plain">from</span><span class="p-Indicator">:</span> <span class="s">"count"</span>
- <span class="l-Scalar-Plain">to</span><span class="p-Indicator">:</span> <span class="s">"log"</span>
- <span class="l-Scalar-Plain">grouping</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">SHUFFLE</span>
+ <span class="pi">-</span> <span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">count</span><span class="nv"> </span><span class="s">--></span><span class="nv"> </span><span class="s">log"</span>
+ <span class="s">from</span><span class="pi">:</span> <span class="s2">"</span><span class="s">count"</span>
+ <span class="s">to</span><span class="pi">:</span> <span class="s2">"</span><span class="s">log"</span>
+ <span class="s">grouping</span><span class="pi">:</span>
+ <span class="s">type</span><span class="pi">:</span> <span class="s">SHUFFLE</span>
</code></pre></div>
-<h2 id="micro-batching-(trident)-api-support">Micro-Batching (Trident) API Support</h2>
+<h2 id="micro-batching-trident-api-support">Micro-Batching (Trident) API Support</h2>
<p>Currenty, the Flux YAML DSL only supports the Core Storm API, but support for Storm's micro-batching API is planned.</p>
<p>To use Flux with a Trident topology, define a topology getter method and reference it in your YAML config:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="s">"my-trident-topology"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">name</span><span class="pi">:</span> <span class="s2">"</span><span class="s">my-trident-topology"</span>
-<span class="l-Scalar-Plain">config</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">topology.workers</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">1</span>
+<span class="s">config</span><span class="pi">:</span>
+ <span class="s">topology.workers</span><span class="pi">:</span> <span class="s">1</span>
-<span class="l-Scalar-Plain">topologySource</span><span class="p-Indicator">:</span>
- <span class="l-Scalar-Plain">className</span><span class="p-Indicator">:</span> <span class="s">"org.apache.storm.flux.test.TridentTopologySource"</span>
- <span class="c1"># Flux will look for "getTopology", this will override that.</span>
- <span class="l-Scalar-Plain">methodName</span><span class="p-Indicator">:</span> <span class="s">"getTopologyWithDifferentMethodName"</span>
+<span class="s">topologySource</span><span class="pi">:</span>
+ <span class="s">className</span><span class="pi">:</span> <span class="s2">"</span><span class="s">org.apache.storm.flux.test.TridentTopologySource"</span>
+ <span class="c1"># Flux will look for "getTopology", this will override that.</span>
+ <span class="s">methodName</span><span class="pi">:</span> <span class="s2">"</span><span class="s">getTopologyWithDifferentMethodName"</span>
</code></pre></div>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/getting-help.html
----------------------------------------------------------------------
diff --git a/_site/documentation/getting-help.html b/_site/documentation/getting-help.html
index f9bba8e..4cc2276 100644
--- a/_site/documentation/getting-help.html
+++ b/_site/documentation/getting-help.html
@@ -106,13 +106,13 @@
<p>You can view the archives of the mailing list <a href="http://mail-archives.apache.org/mod_mbox/storm-dev/">here</a>.</p>
-<h4 id="which-list-should-i-send/subscribe-to?">Which list should I send/subscribe to?</h4>
+<h4 id="which-list-should-i-send-subscribe-to">Which list should I send/subscribe to?</h4>
<p>If you are using a pre-built binary distribution of Storm, then chances are you should send questions, comments, storm-related announcements, etc. to <a href="user@storm.apache.org">user@storm.apache.org</a>. </p>
<p>If you are building storm from source, developing new features, or otherwise hacking storm source code, then <a href="dev@storm.apache.org">dev@storm.apache.org</a> is more appropriate. </p>
-<h4 id="what-will-happen-with-storm-user@googlegroups.com?">What will happen with <a href="mailto:storm-user@googlegroups.com">storm-user@googlegroups.com</a>?</h4>
+<h4 id="what-will-happen-with-storm-user-googlegroups-com">What will happen with <a href="mailto:storm-user@googlegroups.com">storm-user@googlegroups.com</a>?</h4>
<p>All existing messages will remain archived there, and can be accessed/searched <a href="https://groups.google.com/forum/#!forum/storm-user">here</a>.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/storm-eventhubs.html
----------------------------------------------------------------------
diff --git a/_site/documentation/storm-eventhubs.html b/_site/documentation/storm-eventhubs.html
index 17bce90..49cb0aa 100644
--- a/_site/documentation/storm-eventhubs.html
+++ b/_site/documentation/storm-eventhubs.html
@@ -93,19 +93,19 @@
<p>Storm spout and bolt implementation for Microsoft Azure Eventhubs</p>
<h3 id="build">build</h3>
-<div class="highlight"><pre><code class="language-text" data-lang="text">mvn clean package
+<div class="highlight"><pre><code class="language-" data-lang="">mvn clean package
</code></pre></div>
<h3 id="run-sample-topology">run sample topology</h3>
<p>To run the sample topology, you need to modify the config.properties file with
the eventhubs configurations. Here is an example:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">eventhubspout.username = [username: policy name in EventHubs Portal]
+<div class="highlight"><pre><code class="language-" data-lang="">eventhubspout.username = [username: policy name in EventHubs Portal]
eventhubspout.password = [password: shared access key in EventHubs Portal]
eventhubspout.namespace = [namespace]
eventhubspout.entitypath = [entitypath]
eventhubspout.partitions.count = [partitioncount]
-# if not provided, will use storm's zookeeper settings
+# if not provided, will use storm's zookeeper settings
# zookeeper.connectionstring=zookeeper0:2181,zookeeper1:2181,zookeeper2:2181
eventhubspout.checkpoint.interval = 10
@@ -123,7 +123,7 @@ eventhub.receiver.credits = 1024
If you want to send messages to all partitions, use "-1" as partitionId.</p>
<h3 id="windows-azure-eventhubs">Windows Azure Eventhubs</h3>
-<div class="highlight"><pre><code class="language-text" data-lang="text">http://azure.microsoft.com/en-us/services/event-hubs/
+<div class="highlight"><pre><code class="language-" data-lang="">http://azure.microsoft.com/en-us/services/event-hubs/
</code></pre></div>
[07/10] storm git commit: STORM-1366. Add documentation for StormSQL
into the website.
Posted by wh...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Multilang-protocol.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Multilang-protocol.html b/_site/documentation/Multilang-protocol.html
index d1828bd..18e65b8 100644
--- a/_site/documentation/Multilang-protocol.html
+++ b/_site/documentation/Multilang-protocol.html
@@ -141,44 +141,44 @@ STDIN and STDOUT.</li>
<ul>
<li>STDIN: Setup info. This is a JSON object with the Storm configuration, a PID directory, and a topology context, like this:</li>
</ul>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "conf": {
- "topology.message.timeout.secs": 3,
- // etc
- },
- "pidDir": "...",
- "context": {
- "task->component": {
- "1": "example-spout",
- "2": "__acker",
- "3": "example-bolt1",
- "4": "example-bolt2"
- },
- "taskid": 3,
- // Everything below this line is only available in Storm 0.10.0+
- "componentid": "example-bolt"
- "stream->target->grouping": {
- "default": {
- "example-bolt2": {
- "type": "SHUFFLE"}}},
- "streams": ["default"],
- "stream->outputfields": {"default": ["word"]},
- "source->stream->grouping": {
- "example-spout": {
- "default": {
- "type": "FIELDS",
- "fields": ["word"]
- }
- }
- }
- "source->stream->fields": {
- "example-spout": {
- "default": ["word"]
- }
- }
- }
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"conf"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"topology.message.timeout.secs"</span><span class="p">:</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">etc</span><span class="w">
+ </span><span class="err">}</span><span class="p">,</span><span class="w">
+ </span><span class="nt">"pidDir"</span><span class="p">:</span><span class="w"> </span><span class="s2">"..."</span><span class="p">,</span><span class="w">
+ </span><span class="nt">"context"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"task->component"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"1"</span><span class="p">:</span><span class="w"> </span><span class="s2">"example-spout"</span><span class="p">,</span><span class="w">
+ </span><span class="nt">"2"</span><span class="p">:</span><span class="w"> </span><span class="s2">"__acker"</span><span class="p">,</span><span class="w">
+ </span><span class="nt">"3"</span><span class="p">:</span><span class="w"> </span><span class="s2">"example-bolt1"</span><span class="p">,</span><span class="w">
+ </span><span class="nt">"4"</span><span class="p">:</span><span class="w"> </span><span class="s2">"example-bolt2"</span><span class="w">
+ </span><span class="p">},</span><span class="w">
+ </span><span class="nt">"taskid"</span><span class="p">:</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">Everything</span><span class="w"> </span><span class="err">below</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">line</span><span class="w"> </span><span class="err">is</span><span class="w"> </span><span class="err">only</span><span class="w"> </span><span class="err">available</span><span class="w"> </span><span class="err">in</span><span class="w"> </span><span class="err">Storm</span><span class="w"> </span><span class="err">0.10.0+</span><span class="w">
+ </span><span class="nt">"componentid"</span><span class="p">:</span><span class="w"> </span><span class="s2">"example-bolt"</span><span class="w">
+ </span><span class="s2">"stream->target->grouping"</span><span class="err">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"default"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"example-bolt2"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"type"</span><span class="p">:</span><span class="w"> </span><span class="s2">"SHUFFLE"</span><span class="p">}}},</span><span class="w">
+ </span><span class="nt">"streams"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"default"</span><span class="p">],</span><span class="w">
+ </span><span class="nt">"stream->outputfields"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"default"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"word"</span><span class="p">]},</span><span class="w">
+ </span><span class="nt">"source->stream->grouping"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"example-spout"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"default"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"type"</span><span class="p">:</span><span class="w"> </span><span class="s2">"FIELDS"</span><span class="p">,</span><span class="w">
+ </span><span class="nt">"fields"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"word"</span><span class="p">]</span><span class="w">
+ </span><span class="p">}</span><span class="w">
+ </span><span class="p">}</span><span class="w">
+ </span><span class="p">}</span><span class="w">
+ </span><span class="s2">"source->stream->fields"</span><span class="err">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"example-spout"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+ </span><span class="nt">"default"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"word"</span><span class="p">]</span><span class="w">
+ </span><span class="p">}</span><span class="w">
+ </span><span class="p">}</span><span class="w">
+ </span><span class="p">}</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>Your script should create an empty file named with its PID in this directory. e.g.
the PID is 1234, so an empty file named 1234 is created in the directory. This
file lets the supervisor know the PID so it can shutdown the process later on.</p>
@@ -207,47 +207,47 @@ to specify which fields are involved in a <code>FIELDS</code> grouping.</p>
</ul>
<p>"next" is the equivalent of ISpout's <code>nextTuple</code>. It looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{"command": "next"}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"next"</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>"ack" looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{"command": "ack", "id": "1231231"}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"ack"</span><span class="p">,</span><span class="w"> </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1231231"</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>"fail" looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{"command": "fail", "id": "1231231"}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"fail"</span><span class="p">,</span><span class="w"> </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1231231"</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<ul>
<li>STDOUT: The results of your spout for the previous command. This can
be a sequence of emits and logs.</li>
</ul>
<p>An emit looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "emit",
- // The id for the tuple. Leave this out for an unreliable emit. The id can
- // be a string or a number.
- "id": "1231231",
- // The id of the stream this tuple was emitted to. Leave this empty to emit to default stream.
- "stream": "1",
- // If doing an emit direct, indicate the task to send the tuple to
- "task": 9,
- // All the values in this tuple
- "tuple": ["field1", 2, 3]
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"emit"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">for</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuple.</span><span class="w"> </span><span class="err">Leave</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">out</span><span class="w"> </span><span class="err">for</span><span class="w"> </span><span class="err">an</span><span class="w"> </span><span class="err">unreliable</span><span class="w"> </span><span class="err">emit.</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">can</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">be</span><span class="w"> </span><span class="err">a</span><span class="w"> </span><span class="err">string</span><span class="w"> </span><span class="err">or</span><span class="w"> </span><span class="err">a</span><span class="w"> </span><span class="err">number.</span><span class="w">
+ </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1231231"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">stream</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">was</span><span class="w"> </span><span class="err">emitted</span><span class="w"> </span><span class="err">to.</span><span class="w"> </span><span class="err">Leave</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">empty</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">emit</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">default</span><span class="w"> </span><span class="err">stream.</span><sp
an class="w">
+ </span><span class="nt">"stream"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">If</span><span class="w"> </span><span class="err">doing</span><span class="w"> </span><span class="err">an</span><span class="w"> </span><span class="err">emit</span><span class="w"> </span><span class="err">direct,</span><span class="w"> </span><span class="err">indicate</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">task</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">send</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">to</span><span class="w">
+ </span><span class="nt">"task"</span><span class="p">:</span><span class="w"> </span><span class="mi">9</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">All</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">values</span><span class="w"> </span><span class="err">in</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w">
+ </span><span class="nt">"tuple"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"field1"</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">]</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>If not doing an emit direct, you will immediately receive the task ids to which the tuple was emitted on STDIN as a JSON array.</p>
<p>A "log" will log a message in the worker log. It looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "log",
- // the message to log
- "msg": "hello world!"
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"log"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">message</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">log</span><span class="w">
+ </span><span class="nt">"msg"</span><span class="p">:</span><span class="w"> </span><span class="s2">"hello world!"</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<ul>
<li>STDOUT: a "sync" command ends the sequence of emits and logs. It looks like:</li>
</ul>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{"command": "sync"}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"sync"</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>After you sync, ShellSpout will not read your output until it sends another next, ack, or fail command.</p>
<p>Note that, similarly to ISpout, all of the spouts in the worker will be locked up after a next, ack, or fail, until you sync. Also like ISpout, if you have no tuples to emit for a next, you should sleep for a small amount of time before syncing. ShellSpout will not automatically sleep for you.</p>
@@ -259,34 +259,34 @@ be a sequence of emits and logs.</li>
<ul>
<li>STDIN: A tuple! This is a JSON encoded structure like this:</li>
</ul>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- // The tuple's id - this is a string to support languages lacking 64-bit precision
- "id": "-6955786537413359385",
- // The id of the component that created this tuple
- "comp": "1",
- // The id of the stream this tuple was emitted to
- "stream": "1",
- // The id of the task that created this tuple
- "task": 9,
- // All the values in this tuple
- "tuple": ["snow white and the seven dwarfs", "field2", 3]
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">tuple's</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">-</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">is</span><span class="w"> </span><span class="err">a</span><span class="w"> </span><span class="err">string</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">support</span><span class="w"> </span><span class="err">languages</span><span class="w"> </span><span class="err">lacking</span><span class="w"> </span><span class="err">64-bit</span><span class="w"> </span><span class="err">precision</span><span class="w">
+ </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"-6955786537413359385"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">component</span><span class="w"> </span><span class="err">that</span><span class="w"> </span><span class="err">created</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w">
+ </span><span class="nt">"comp"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">stream</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">was</span><span class="w"> </span><span class="err">emitted</span><span class="w"> </span><span class="err">to</span><span class="w">
+ </span><span class="nt">"stream"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">task</span><span class="w"> </span><span class="err">that</span><span class="w"> </span><span class="err">created</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w">
+ </span><span class="nt">"task"</span><span class="p">:</span><span class="w"> </span><span class="mi">9</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">All</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">values</span><span class="w"> </span><span class="err">in</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w">
+ </span><span class="nt">"tuple"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"snow white and the seven dwarfs"</span><span class="p">,</span><span class="w"> </span><span class="s2">"field2"</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">]</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<ul>
<li>STDOUT: An ack, fail, emit, or log. Emits look like:</li>
</ul>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "emit",
- // The ids of the tuples this output tuples should be anchored to
- "anchors": ["1231231", "-234234234"],
- // The id of the stream this tuple was emitted to. Leave this empty to emit to default stream.
- "stream": "1",
- // If doing an emit direct, indicate the task to send the tuple to
- "task": 9,
- // All the values in this tuple
- "tuple": ["field1", 2, 3]
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"emit"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">ids</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuples</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">output</span><span class="w"> </span><span class="err">tuples</span><span class="w"> </span><span class="err">should</span><span class="w"> </span><span class="err">be</span><span class="w"> </span><span class="err">anchored</span><span class="w"> </span><span class="err">to</span><span class="w">
+ </span><span class="nt">"anchors"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"1231231"</span><span class="p">,</span><span class="w"> </span><span class="s2">"-234234234"</span><span class="p">],</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">stream</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">was</span><span class="w"> </span><span class="err">emitted</span><span class="w"> </span><span class="err">to.</span><span class="w"> </span><span class="err">Leave</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">empty</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">emit</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">default</span><span class="w"> </span><span class="err">stream.</span><sp
an class="w">
+ </span><span class="nt">"stream"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">If</span><span class="w"> </span><span class="err">doing</span><span class="w"> </span><span class="err">an</span><span class="w"> </span><span class="err">emit</span><span class="w"> </span><span class="err">direct,</span><span class="w"> </span><span class="err">indicate</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">task</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">send</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">to</span><span class="w">
+ </span><span class="nt">"task"</span><span class="p">:</span><span class="w"> </span><span class="mi">9</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">All</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">values</span><span class="w"> </span><span class="err">in</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w">
+ </span><span class="nt">"tuple"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"field1"</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">]</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>If not doing an emit direct, you will receive the task ids to which
the tuple was emitted on STDIN as a JSON array. Note that, due to the
asynchronous nature of the shell bolt protocol, when you read after
@@ -296,32 +296,32 @@ receive the task id lists in the same order as their corresponding
emits, however.</p>
<p>An ack looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "ack",
- // the id of the tuple to ack
- "id": "123123"
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"ack"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">ack</span><span class="w">
+ </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"123123"</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>A fail looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "fail",
- // the id of the tuple to fail
- "id": "123123"
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"fail"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">fail</span><span class="w">
+ </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"123123"</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>A "log" will log a message in the worker log. It looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "log",
- // the message to log
- "msg": "hello world!"
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"log"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">message</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">log</span><span class="w">
+ </span><span class="nt">"msg"</span><span class="p">:</span><span class="w"> </span><span class="s2">"hello world!"</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<ul>
<li>Note that, as of version 0.7.1, there is no longer any need for a
shell bolt to 'sync'.</li>
</ul>
-<h3 id="handling-heartbeats-(0.9.3-and-later)">Handling Heartbeats (0.9.3 and later)</h3>
+<h3 id="handling-heartbeats-0-9-3-and-later">Handling Heartbeats (0.9.3 and later)</h3>
<p>As of Storm 0.9.3, heartbeats have been between ShellSpout/ShellBolt and their
multi-lang subprocesses to detect hanging/zombie subprocesses. Any libraries
@@ -339,15 +339,15 @@ timeout during <code>next()</code>.</p>
<p>Shell bolts are asynchronous, so a ShellBolt will send heartbeat tuples to its
subprocess periodically. Heartbeat tuple looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "id": "-6955786537413359385",
- "comp": "1",
- "stream": "__heartbeat",
- // this shell bolt's system task id
- "task": -1,
- "tuple": []
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"-6955786537413359385"</span><span class="p">,</span><span class="w">
+ </span><span class="nt">"comp"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1"</span><span class="p">,</span><span class="w">
+ </span><span class="nt">"stream"</span><span class="p">:</span><span class="w"> </span><span class="s2">"__heartbeat"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">shell</span><span class="w"> </span><span class="err">bolt's</span><span class="w"> </span><span class="err">system</span><span class="w"> </span><span class="err">task</span><span class="w"> </span><span class="err">id</span><span class="w">
+ </span><span class="nt">"task"</span><span class="p">:</span><span class="w"> </span><span class="mi">-1</span><span class="p">,</span><span class="w">
+ </span><span class="nt">"tuple"</span><span class="p">:</span><span class="w"> </span><span class="p">[]</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>When subprocess receives heartbeat tuple, it must send a <code>sync</code> command back to
ShellBolt.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Running-topologies-on-a-production-cluster.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Running-topologies-on-a-production-cluster.html b/_site/documentation/Running-topologies-on-a-production-cluster.html
index 5cd4b12..8ada823 100644
--- a/_site/documentation/Running-topologies-on-a-production-cluster.html
+++ b/_site/documentation/Running-topologies-on-a-production-cluster.html
@@ -95,10 +95,10 @@
<p>1) Define the topology (Use <a href="/javadoc/apidocs/backtype/storm/topology/TopologyBuilder.html">TopologyBuilder</a> if defining using Java)</p>
<p>2) Use <a href="/javadoc/apidocs/backtype/storm/StormSubmitter.html">StormSubmitter</a> to submit the topology to the cluster. <code>StormSubmitter</code> takes as input the name of the topology, a configuration for the topology, and the topology itself. For example:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Config</span> <span class="n">conf</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Config</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Config</span> <span class="n">conf</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Config</span><span class="o">();</span>
<span class="n">conf</span><span class="o">.</span><span class="na">setNumWorkers</span><span class="o">(</span><span class="mi">20</span><span class="o">);</span>
<span class="n">conf</span><span class="o">.</span><span class="na">setMaxSpoutPending</span><span class="o">(</span><span class="mi">5000</span><span class="o">);</span>
-<span class="n">StormSubmitter</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"mytopology"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">topology</span><span class="o">);</span>
+<span class="n">StormSubmitter</span><span class="o">.</span><span class="na">submitTopology</span><span class="o">(</span><span class="s">"mytopology"</span><span class="o">,</span> <span class="n">conf</span><span class="o">,</span> <span class="n">topology</span><span class="o">);</span>
</code></pre></div>
<p>3) Create a jar containing your code and all the dependencies of your code (except for Storm -- the Storm jars will be added to the classpath on the worker nodes).</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Serialization-(prior-to-0.6.0).html
----------------------------------------------------------------------
diff --git a/_site/documentation/Serialization-(prior-to-0.6.0).html b/_site/documentation/Serialization-(prior-to-0.6.0).html
index fa8f93c..0780464 100644
--- a/_site/documentation/Serialization-(prior-to-0.6.0).html
+++ b/_site/documentation/Serialization-(prior-to-0.6.0).html
@@ -112,9 +112,9 @@
<p>The interface looks like this:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">ISerialization</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="o">{</span>
- <span class="kd">public</span> <span class="kt">boolean</span> <span class="nf">accept</span><span class="o">(</span><span class="n">Class</span> <span class="n">c</span><span class="o">);</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">serialize</span><span class="o">(</span><span class="n">T</span> <span class="n">object</span><span class="o">,</span> <span class="n">DataOutputStream</span> <span class="n">stream</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">IOException</span><span class="o">;</span>
- <span class="kd">public</span> <span class="n">T</span> <span class="nf">deserialize</span><span class="o">(</span><span class="n">DataInputStream</span> <span class="n">stream</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">IOException</span><span class="o">;</span>
+ <span class="kd">public</span> <span class="kt">boolean</span> <span class="n">accept</span><span class="o">(</span><span class="n">Class</span> <span class="n">c</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">serialize</span><span class="o">(</span><span class="n">T</span> <span class="n">object</span><span class="o">,</span> <span class="n">DataOutputStream</span> <span class="n">stream</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">IOException</span><span class="o">;</span>
+ <span class="kd">public</span> <span class="n">T</span> <span class="n">deserialize</span><span class="o">(</span><span class="n">DataInputStream</span> <span class="n">stream</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">IOException</span><span class="o">;</span>
<span class="o">}</span>
</code></pre></div>
<p>Storm uses the <code>accept</code> method to determine if a type can be serialized by this serializer. Remember, Storm's tuples are dynamically typed so Storm determines what serializer to use at runtime.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Serialization.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Serialization.html b/_site/documentation/Serialization.html
index fa78754..9374f0a 100644
--- a/_site/documentation/Serialization.html
+++ b/_site/documentation/Serialization.html
@@ -94,7 +94,7 @@
<p>Tuples can be comprised of objects of any types. Since Storm is a distributed system, it needs to know how to serialize and deserialize objects when they're passed between tasks.</p>
-<p>Storm uses <a href="http://code.google.com/p/kryo/">Kryo</a> for serialization. Kryo is a flexible and fast serialization library that produces small serializations.</p>
+<p>Storm uses <a href="https://github.com/EsotericSoftware/kryo">Kryo</a> for serialization. Kryo is a flexible and fast serialization library that produces small serializations.</p>
<p>By default, Storm can serialize primitive types, strings, byte arrays, ArrayList, HashMap, HashSet, and the Clojure collection types. If you want to use another type in your tuples, you'll need to register a custom serializer.</p>
@@ -110,17 +110,17 @@
<h3 id="custom-serialization">Custom serialization</h3>
-<p>As mentioned, Storm uses Kryo for serialization. To implement custom serializers, you need to register new serializers with Kryo. It's highly recommended that you read over <a href="http://code.google.com/p/kryo/">Kryo's home page</a> to understand how it handles custom serialization.</p>
+<p>As mentioned, Storm uses Kryo for serialization. To implement custom serializers, you need to register new serializers with Kryo. It's highly recommended that you read over <a href="https://github.com/EsotericSoftware/kryo">Kryo's home page</a> to understand how it handles custom serialization.</p>
<p>Adding custom serializers is done through the "topology.kryo.register" property in your topology config. It takes a list of registrations, where each registration can take one of two forms:</p>
<ol>
<li>The name of a class to register. In this case, Storm will use Kryo's <code>FieldsSerializer</code> to serialize the class. This may or may not be optimal for the class -- see the Kryo docs for more details.</li>
-<li>A map from the name of a class to register to an implementation of <a href="http://code.google.com/p/kryo/source/browse/trunk/src/com/esotericsoftware/kryo/Serializer.java">com.esotericsoftware.kryo.Serializer</a>.</li>
+<li>A map from the name of a class to register to an implementation of <a href="https://github.com/EsotericSoftware/kryo/blob/master/src/com/esotericsoftware/kryo/Serializer.java">com.esotericsoftware.kryo.Serializer</a>.</li>
</ol>
<p>Let's look at an example.</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">topology.kryo.register:
+<div class="highlight"><pre><code class="language-" data-lang="">topology.kryo.register:
- com.mycompany.CustomType1
- com.mycompany.CustomType2: com.mycompany.serializer.CustomType2Serializer
- com.mycompany.CustomType3
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Setting-up-a-Storm-cluster.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Setting-up-a-Storm-cluster.html b/_site/documentation/Setting-up-a-Storm-cluster.html
index 966d3d4..a7167b4 100644
--- a/_site/documentation/Setting-up-a-Storm-cluster.html
+++ b/_site/documentation/Setting-up-a-Storm-cluster.html
@@ -130,35 +130,35 @@
<p>Next, download a Storm release and extract the zip file somewhere on Nimbus and each of the worker machines. The Storm releases can be downloaded <a href="http://github.com/apache/storm/releases">from here</a>.</p>
-<h3 id="fill-in-mandatory-configurations-into-storm.yaml">Fill in mandatory configurations into storm.yaml</h3>
+<h3 id="fill-in-mandatory-configurations-into-storm-yaml">Fill in mandatory configurations into storm.yaml</h3>
<p>The Storm release contains a file at <code>conf/storm.yaml</code> that configures the Storm daemons. You can see the default configuration values <a href="https://github.com/apache/storm/blob/master/conf/defaults.yaml">here</a>. storm.yaml overrides anything in defaults.yaml. There's a few configurations that are mandatory to get a working cluster:</p>
<p>1) <strong>storm.zookeeper.servers</strong>: This is a list of the hosts in the Zookeeper cluster for your Storm cluster. It should look something like:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">storm.zookeeper.servers</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="s">"111.222.333.444"</span>
- <span class="p-Indicator">-</span> <span class="s">"555.666.777.888"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">storm.zookeeper.servers</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">111.222.333.444"</span>
+ <span class="pi">-</span> <span class="s2">"</span><span class="s">555.666.777.888"</span>
</code></pre></div>
<p>If the port that your Zookeeper cluster uses is different than the default, you should set <strong>storm.zookeeper.port</strong> as well.</p>
<p>2) <strong>storm.local.dir</strong>: The Nimbus and Supervisor daemons require a directory on the local disk to store small amounts of state (like jars, confs, and things like that). You should create that directory on each machine, give it proper permissions, and then fill in the directory location using this config. For example:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">storm.local.dir</span><span class="p-Indicator">:</span> <span class="s">"/mnt/storm"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">storm.local.dir</span><span class="pi">:</span> <span class="s2">"</span><span class="s">/mnt/storm"</span>
</code></pre></div>
<p>3) <strong>nimbus.host</strong>: The worker nodes need to know which machine is the master in order to download topology jars and confs. For example:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">nimbus.host</span><span class="p-Indicator">:</span> <span class="s">"111.222.333.44"</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">nimbus.host</span><span class="pi">:</span> <span class="s2">"</span><span class="s">111.222.333.44"</span>
</code></pre></div>
<p>4) <strong>supervisor.slots.ports</strong>: For each worker machine, you configure how many workers run on that machine with this config. Each worker uses a single port for receiving messages, and this setting defines which ports are open for use. If you define five ports here, then Storm will allocate up to five workers to run on this machine. If you define three ports, Storm will only run up to three. By default, this setting is configured to run 4 workers on the ports 6700, 6701, 6702, and 6703. For example:</p>
-<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">supervisor.slots.ports</span><span class="p-Indicator">:</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">6700</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">6701</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">6702</span>
- <span class="p-Indicator">-</span> <span class="l-Scalar-Plain">6703</span>
+<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="s">supervisor.slots.ports</span><span class="pi">:</span>
+ <span class="pi">-</span> <span class="s">6700</span>
+ <span class="pi">-</span> <span class="s">6701</span>
+ <span class="pi">-</span> <span class="s">6702</span>
+ <span class="pi">-</span> <span class="s">6703</span>
</code></pre></div>
-<h3 id="configure-external-libraries-and-environmental-variables-(optional)">Configure external libraries and environmental variables (optional)</h3>
+<h3 id="configure-external-libraries-and-environmental-variables-optional">Configure external libraries and environmental variables (optional)</h3>
<p>If you need support from external libraries or custom plugins, you can place such jars into the extlib/ and extlib-daemon/ directories. Note that the extlib-daemon/ directory stores jars used only by daemons (Nimbus, Supervisor, DRPC, UI, Logviewer), e.g., HDFS and customized scheduling libraries. Accordingly, two environmental variables STORM_EXT_CLASSPATH and STORM_EXT_CLASSPATH_DAEMON can be configured by users for including the external classpath and daemon-only external classpath.</p>
-<h3 id="launch-daemons-under-supervision-using-"storm"-script-and-a-supervisor-of-your-choice">Launch daemons under supervision using "storm" script and a supervisor of your choice</h3>
+<h3 id="launch-daemons-under-supervision-using-quot-storm-quot-script-and-a-supervisor-of-your-choice">Launch daemons under supervision using "storm" script and a supervisor of your choice</h3>
<p>The last step is to launch all the Storm daemons. It is critical that you run each of these daemons under supervision. Storm is a <strong>fail-fast</strong> system which means the processes will halt whenever an unexpected error is encountered. Storm is designed so that it can safely halt at any point and recover correctly when the process is restarted. This is why Storm keeps no state in-process -- if Nimbus or the Supervisors restart, the running topologies are unaffected. Here's how to run the Storm daemons:</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Setting-up-development-environment.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Setting-up-development-environment.html b/_site/documentation/Setting-up-development-environment.html
index 8a015ae..082e547 100644
--- a/_site/documentation/Setting-up-development-environment.html
+++ b/_site/documentation/Setting-up-development-environment.html
@@ -99,7 +99,7 @@
<p>More detail on each of these steps is below.</p>
-<h3 id="what-is-a-development-environment?">What is a development environment?</h3>
+<h3 id="what-is-a-development-environment">What is a development environment?</h3>
<p>Storm has two modes of operation: local mode and remote mode. In local mode, you can develop and test topologies completely in process on your local machine. In remote mode, you submit topologies for execution on a cluster of machines.</p>
@@ -116,10 +116,10 @@
<h3 id="starting-and-stopping-topologies-on-a-remote-cluster">Starting and stopping topologies on a remote cluster</h3>
<p>The previous step installed the <code>storm</code> client on your machine which is used to communicate with remote Storm clusters. Now all you have to do is tell the client which Storm cluster to talk to. To do this, all you have to do is put the host address of the master in the <code>~/.storm/storm.yaml</code> file. It should look something like this:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">nimbus.host: "123.45.678.890"
+<div class="highlight"><pre><code class="language-" data-lang="">nimbus.host: "123.45.678.890"
</code></pre></div>
<p>Alternatively, if you use the <a href="https://github.com/nathanmarz/storm-deploy">storm-deploy</a> project to provision Storm clusters on AWS, it will automatically set up your ~/.storm/storm.yaml file. You can manually attach to a Storm cluster (or switch between multiple clusters) using the "attach" command, like so:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">lein run :deploy --attach --name mystormcluster
+<div class="highlight"><pre><code class="language-" data-lang="">lein run :deploy --attach --name mystormcluster
</code></pre></div>
<p>More information is on the storm-deploy <a href="https://github.com/nathanmarz/storm-deploy/wiki">wiki</a></p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Storm-multi-language-protocol-(versions-0.7.0-and-below).html
----------------------------------------------------------------------
diff --git a/_site/documentation/Storm-multi-language-protocol-(versions-0.7.0-and-below).html b/_site/documentation/Storm-multi-language-protocol-(versions-0.7.0-and-below).html
index 83bb4ff..86a966d 100644
--- a/_site/documentation/Storm-multi-language-protocol-(versions-0.7.0-and-below).html
+++ b/_site/documentation/Storm-multi-language-protocol-(versions-0.7.0-and-below).html
@@ -140,56 +140,56 @@ file lets the supervisor know the PID so it can shutdown the process later on.</
<li>The rest happens in a while(true) loop</li>
<li>STDIN: A tuple! This is a JSON encoded structure like this:</li>
</ul>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- // The tuple's id
- "id": -6955786537413359385,
- // The id of the component that created this tuple
- "comp": 1,
- // The id of the stream this tuple was emitted to
- "stream": 1,
- // The id of the task that created this tuple
- "task": 9,
- // All the values in this tuple
- "tuple": ["snow white and the seven dwarfs", "field2", 3]
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">tuple's</span><span class="w"> </span><span class="err">id</span><span class="w">
+ </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="mi">-6955786537413359385</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">component</span><span class="w"> </span><span class="err">that</span><span class="w"> </span><span class="err">created</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w">
+ </span><span class="nt">"comp"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">stream</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">was</span><span class="w"> </span><span class="err">emitted</span><span class="w"> </span><span class="err">to</span><span class="w">
+ </span><span class="nt">"stream"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">task</span><span class="w"> </span><span class="err">that</span><span class="w"> </span><span class="err">created</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w">
+ </span><span class="nt">"task"</span><span class="p">:</span><span class="w"> </span><span class="mi">9</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">All</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">values</span><span class="w"> </span><span class="err">in</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w">
+ </span><span class="nt">"tuple"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"snow white and the seven dwarfs"</span><span class="p">,</span><span class="w"> </span><span class="s2">"field2"</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">]</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<ul>
<li>STDOUT: The results of your bolt, JSON encoded. This can be a sequence of acks, fails, emits, and/or logs. Emits look like:</li>
</ul>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "emit",
- // The ids of the tuples this output tuples should be anchored to
- "anchors": [1231231, -234234234],
- // The id of the stream this tuple was emitted to. Leave this empty to emit to default stream.
- "stream": 1,
- // If doing an emit direct, indicate the task to sent the tuple to
- "task": 9,
- // All the values in this tuple
- "tuple": ["field1", 2, 3]
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"emit"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">ids</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuples</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">output</span><span class="w"> </span><span class="err">tuples</span><span class="w"> </span><span class="err">should</span><span class="w"> </span><span class="err">be</span><span class="w"> </span><span class="err">anchored</span><span class="w"> </span><span class="err">to</span><span class="w">
+ </span><span class="nt">"anchors"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="mi">1231231</span><span class="p">,</span><span class="w"> </span><span class="mi">-234234234</span><span class="p">],</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">The</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">stream</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">was</span><span class="w"> </span><span class="err">emitted</span><span class="w"> </span><span class="err">to.</span><span class="w"> </span><span class="err">Leave</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">empty</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">emit</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">default</span><span class="w"> </span><span class="err">stream.</span><sp
an class="w">
+ </span><span class="nt">"stream"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">If</span><span class="w"> </span><span class="err">doing</span><span class="w"> </span><span class="err">an</span><span class="w"> </span><span class="err">emit</span><span class="w"> </span><span class="err">direct,</span><span class="w"> </span><span class="err">indicate</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">task</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">sent</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">to</span><span class="w">
+ </span><span class="nt">"task"</span><span class="p">:</span><span class="w"> </span><span class="mi">9</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">All</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">values</span><span class="w"> </span><span class="err">in</span><span class="w"> </span><span class="err">this</span><span class="w"> </span><span class="err">tuple</span><span class="w">
+ </span><span class="nt">"tuple"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"field1"</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">]</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>An ack looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "ack",
- // the id of the tuple to ack
- "id": 123123
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"ack"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">ack</span><span class="w">
+ </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="mi">123123</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>A fail looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "fail",
- // the id of the tuple to fail
- "id": 123123
-}
-</code></pre></div>
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"fail"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">id</span><span class="w"> </span><span class="err">of</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">tuple</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">fail</span><span class="w">
+ </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="mi">123123</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<p>A "log" will log a message in the worker log. It looks like:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">{
- "command": "log",
- // the message to log
- "msg": "hello world!"
+<div class="highlight"><pre><code class="language-" data-lang=""><span class="p">{</span><span class="w">
+ </span><span class="nt">"command"</span><span class="p">:</span><span class="w"> </span><span class="s2">"log"</span><span class="p">,</span><span class="w">
+ </span><span class="err">//</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">message</span><span class="w"> </span><span class="err">to</span><span class="w"> </span><span class="err">log</span><span class="w">
+ </span><span class="nt">"msg"</span><span class="p">:</span><span class="w"> </span><span class="s2">"hello world!"</span><span class="w">
-}
-</code></pre></div>
+</span><span class="p">}</span><span class="w">
+</span></code></pre></div>
<ul>
<li>STDOUT: emit "sync" as a single line by itself when the bolt has finished emitting/acking/failing and is ready for the next input</li>
</ul>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Structure-of-the-codebase.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Structure-of-the-codebase.html b/_site/documentation/Structure-of-the-codebase.html
index a0d3648..6b8a63e 100644
--- a/_site/documentation/Structure-of-the-codebase.html
+++ b/_site/documentation/Structure-of-the-codebase.html
@@ -100,7 +100,7 @@
<p>The following sections explain each of these layers in more detail.</p>
-<h3 id="storm.thrift">storm.thrift</h3>
+<h3 id="storm-thrift">storm.thrift</h3>
<p>The first place to look to understand the structure of Storm's codebase is the <a href="https://github.com/apache/storm/blob/master/storm-core/src/storm.thrift">storm.thrift</a> file.</p>
@@ -172,7 +172,7 @@
<p><a href="https://github.com/apache/storm/tree/master/storm-core/src/jvm/backtype/storm/hooks">backtype.storm.hooks</a>: Interfaces for hooking into various events in Storm, such as when tasks emit tuples, when tuples are acked, etc. User guide for hooks is <a href="https://github.com/apache/storm/wiki/Hooks">here</a>.</p>
-<p><a href="https://github.com/apache/storm/tree/master/storm-core/src/jvm/backtype/storm/serialization">backtype.storm.serialization</a>: Implementation of how Storm serializes/deserializes tuples. Built on top of <a href="http://code.google.com/p/kryo/">Kryo</a>.</p>
+<p><a href="https://github.com/apache/storm/tree/master/storm-core/src/jvm/backtype/storm/serialization">backtype.storm.serialization</a>: Implementation of how Storm serializes/deserializes tuples. Built on top of <a href="https://github.com/EsotericSoftware/kryo">Kryo</a>.</p>
<p><a href="https://github.com/apache/storm/tree/master/storm-core/src/jvm/backtype/storm/spout">backtype.storm.spout</a>: Definition of spout and associated interfaces (like the <code>SpoutOutputCollector</code>). Also contains <code>ShellSpout</code> which implements the protocol for defining spouts in non-JVM languages.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/Transactional-topologies.html
----------------------------------------------------------------------
diff --git a/_site/documentation/Transactional-topologies.html b/_site/documentation/Transactional-topologies.html
index 433f812..5d757a0 100644
--- a/_site/documentation/Transactional-topologies.html
+++ b/_site/documentation/Transactional-topologies.html
@@ -141,7 +141,7 @@
<p>After bolt 1 finishes its portion of the processing, it will be idle until the rest of the bolts finish and the next batch can be emitted from the spout.</p>
-<h3 id="design-3-(storm's-design)">Design 3 (Storm's design)</h3>
+<h3 id="design-3-storm-39-s-design">Design 3 (Storm's design)</h3>
<p>A key realization is that not all the work for processing batches of tuples needs to be strongly ordered. For example, when computing a global count, there's two parts to the computation:</p>
@@ -177,12 +177,12 @@
<h2 id="the-basics-through-example">The basics through example</h2>
<p>You build transactional topologies by using <a href="/javadoc/apidocs/backtype/storm/transactional/TransactionalTopologyBuilder.html">TransactionalTopologyBuilder</a>. Here's the transactional topology definition for a topology that computes the global count of tuples from the input stream. This code comes from <a href="https://github.com/apache/storm/blob/master/examples/storm-starter/src/jvm/storm/starter/TransactionalGlobalCount.java">TransactionalGlobalCount</a> in storm-starter.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">MemoryTransactionalSpout</span> <span class="n">spout</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">MemoryTransactionalSpout</span><span class="o">(</span><span class="n">DATA</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">),</span> <span class="n">PARTITION_TAKE_PER_BATCH</span><span class="o">);</span>
-<span class="n">TransactionalTopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TransactionalTopologyBuilder</span><span class="o">(</span><span class="s">"global-count"</span><span class="o">,</span> <span class="s">"spout"</span><span class="o">,</span> <span class="n">spout</span><span class="o">,</span> <span class="mi">3</span><span class="o">);</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"partial-count"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">BatchCount</span><span class="o">(),</span> <span class="mi">5</span><span class="o">)</span>
- <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"spout"</span><span class="o">);</span>
-<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"sum"</span><span class="o">,</span> <span class="k">new</span> <span class="nf">UpdateGlobalCount</span><span class="o">())</span>
- <span class="o">.</span><span class="na">globalGrouping</span><span class="o">(</span><span class="s">"partial-count"</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">MemoryTransactionalSpout</span> <span class="n">spout</span> <span class="o">=</span> <span class="k">new</span> <span class="n">MemoryTransactionalSpout</span><span class="o">(</span><span class="n">DATA</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">),</span> <span class="n">PARTITION_TAKE_PER_BATCH</span><span class="o">);</span>
+<span class="n">TransactionalTopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TransactionalTopologyBuilder</span><span class="o">(</span><span class="s">"global-count"</span><span class="o">,</span> <span class="s">"spout"</span><span class="o">,</span> <span class="n">spout</span><span class="o">,</span> <span class="mi">3</span><span class="o">);</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"partial-count"</span><span class="o">,</span> <span class="k">new</span> <span class="n">BatchCount</span><span class="o">(),</span> <span class="mi">5</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"spout"</span><span class="o">);</span>
+<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"sum"</span><span class="o">,</span> <span class="k">new</span> <span class="n">UpdateGlobalCount</span><span class="o">())</span>
+ <span class="o">.</span><span class="na">globalGrouping</span><span class="o">(</span><span class="s">"partial-count"</span><span class="o">);</span>
</code></pre></div>
<p><code>TransactionalTopologyBuilder</code> takes as input in the constructor an id for the transactional topology, an id for the spout within the topology, a transactional spout, and optionally the parallelism for the transactional spout. The id for the transactional topology is used to store state about the progress of topology in Zookeeper, so that if you restart the topology it will continue where it left off.</p>
@@ -198,24 +198,24 @@
<span class="kt">int</span> <span class="n">_count</span> <span class="o">=</span> <span class="mi">0</span><span class="o">;</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">BatchOutputCollector</span> <span class="n">collector</span><span class="o">,</span> <span class="n">Object</span> <span class="n">id</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">BatchOutputCollector</span> <span class="n">collector</span><span class="o">,</span> <span class="n">Object</span> <span class="n">id</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_collector</span> <span class="o">=</span> <span class="n">collector</span><span class="o">;</span>
<span class="n">_id</span> <span class="o">=</span> <span class="n">id</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_count</span><span class="o">++;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">finishBatch</span><span class="o">()</span> <span class="o">{</span>
- <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">_id</span><span class="o">,</span> <span class="n">_count</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">finishBatch</span><span class="o">()</span> <span class="o">{</span>
+ <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">_id</span><span class="o">,</span> <span class="n">_count</span><span class="o">));</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"count"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"count"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
@@ -243,22 +243,22 @@
<span class="kt">int</span> <span class="n">_sum</span> <span class="o">=</span> <span class="mi">0</span><span class="o">;</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">BatchOutputCollector</span> <span class="n">collector</span><span class="o">,</span> <span class="n">TransactionAttempt</span> <span class="n">attempt</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">BatchOutputCollector</span> <span class="n">collector</span><span class="o">,</span> <span class="n">TransactionAttempt</span> <span class="n">attempt</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_collector</span> <span class="o">=</span> <span class="n">collector</span><span class="o">;</span>
<span class="n">_attempt</span> <span class="o">=</span> <span class="n">attempt</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="n">_sum</span><span class="o">+=</span><span class="n">tuple</span><span class="o">.</span><span class="na">getInteger</span><span class="o">(</span><span class="mi">1</span><span class="o">);</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">finishBatch</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">finishBatch</span><span class="o">()</span> <span class="o">{</span>
<span class="n">Value</span> <span class="n">val</span> <span class="o">=</span> <span class="n">DATABASE</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="n">GLOBAL_COUNT_KEY</span><span class="o">);</span>
<span class="n">Value</span> <span class="n">newval</span><span class="o">;</span>
<span class="k">if</span><span class="o">(</span><span class="n">val</span> <span class="o">==</span> <span class="kc">null</span> <span class="o">||</span> <span class="o">!</span><span class="n">val</span><span class="o">.</span><span class="na">txid</span><span class="o">.</span><span class="na">equals</span><span class="o">(</span><span class="n">_attempt</span><span class="o">.</span><span class="na">getTransactionId</span><span class="o">()))</span> <span class="o">{</span>
- <span class="n">newval</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Value</span><span class="o">();</span>
+ <span class="n">newval</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Value</span><span class="o">();</span>
<span class="n">newval</span><span class="o">.</span><span class="na">txid</span> <span class="o">=</span> <span class="n">_attempt</span><span class="o">.</span><span class="na">getTransactionId</span><span class="o">();</span>
<span class="k">if</span><span class="o">(</span><span class="n">val</span><span class="o">==</span><span class="kc">null</span><span class="o">)</span> <span class="o">{</span>
<span class="n">newval</span><span class="o">.</span><span class="na">count</span> <span class="o">=</span> <span class="n">_sum</span><span class="o">;</span>
@@ -269,12 +269,12 @@
<span class="o">}</span> <span class="k">else</span> <span class="o">{</span>
<span class="n">newval</span> <span class="o">=</span> <span class="n">val</span><span class="o">;</span>
<span class="o">}</span>
- <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">_attempt</span><span class="o">,</span> <span class="n">newval</span><span class="o">.</span><span class="na">count</span><span class="o">));</span>
+ <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">_attempt</span><span class="o">,</span> <span class="n">newval</span><span class="o">.</span><span class="na">count</span><span class="o">));</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"sum"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"id"</span><span class="o">,</span> <span class="s">"sum"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
</code></pre></div>
@@ -300,7 +300,7 @@
<li>BatchBolt's that are marked as committers: The only difference between this bolt and a regular batch bolt is when <code>finishBatch</code> is called. A committer bolt has <code>finishedBatch</code> called during the commit phase. The commit phase is guaranteed to occur only after all prior batches have successfully committed, and it will be retried until all bolts in the topology succeed the commit for the batch. There are two ways to make a <code>BatchBolt</code> a committer, by having the <code>BatchBolt</code> implement the <a href="/javadoc/apidocs/backtype/storm/transactional/ICommitter.html">ICommitter</a> marker interface, or by using the <code>setCommiterBolt</code> method in <code>TransactionalTopologyBuilder</code>.</li>
</ol>
-<h4 id="processing-phase-vs.-commit-phase-in-bolts">Processing phase vs. commit phase in bolts</h4>
+<h4 id="processing-phase-vs-commit-phase-in-bolts">Processing phase vs. commit phase in bolts</h4>
<p>To nail down the difference between the processing phase and commit phase of a transaction, let's look at an example topology:</p>
@@ -351,7 +351,7 @@
<li><em>Number of active batches permissible at once:</em> You must set a limit to the number of batches that can be processed at once. You configure this using the "topology.max.spout.pending" config. If you don't set this config, it will default to 1.</li>
</ol>
-<h2 id="what-if-you-can't-emit-the-same-batch-of-tuples-for-a-given-transaction-id?">What if you can't emit the same batch of tuples for a given transaction id?</h2>
+<h2 id="what-if-you-can-39-t-emit-the-same-batch-of-tuples-for-a-given-transaction-id">What if you can't emit the same batch of tuples for a given transaction id?</h2>
<p>So far the discussion around transactional topologies has assumed that you can always emit the exact same batch of tuples for the same transaction id. So what do you do if this is not possible?</p>
[02/10] storm git commit: STORM-1366. Add documentation for StormSQL
into the website.
Posted by wh...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/storm-jdbc.html
----------------------------------------------------------------------
diff --git a/_site/documentation/storm-jdbc.html b/_site/documentation/storm-jdbc.html
index 6bcd480..f95c1ee 100644
--- a/_site/documentation/storm-jdbc.html
+++ b/_site/documentation/storm-jdbc.html
@@ -96,7 +96,7 @@ in a storm topology.</p>
<p><strong>Note</strong>: Throughout the examples below, we make use of com.google.common.collect.Lists and com.google.common.collect.Maps.</p>
-<h2 id="inserting-into-a-database.">Inserting into a database.</h2>
+<h2 id="inserting-into-a-database">Inserting into a database.</h2>
<p>The bolt and trident state included in this package for inserting data into a database tables are tied to a single table.</p>
@@ -104,21 +104,21 @@ in a storm topology.</p>
<p>An interface that should be implemented by different connection pooling mechanism <code>org.apache.storm.jdbc.common.ConnectionProvider</code></p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">ConnectionProvider</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="cm">/**</span>
-<span class="cm"> * method must be idempotent.</span>
-<span class="cm"> */</span>
- <span class="kt">void</span> <span class="nf">prepare</span><span class="o">();</span>
-
- <span class="cm">/**</span>
-<span class="cm"> *</span>
-<span class="cm"> * @return a DB connection over which the queries can be executed.</span>
-<span class="cm"> */</span>
- <span class="n">Connection</span> <span class="nf">getConnection</span><span class="o">();</span>
-
- <span class="cm">/**</span>
-<span class="cm"> * called once when the system is shutting down, should be idempotent.</span>
-<span class="cm"> */</span>
- <span class="kt">void</span> <span class="nf">cleanup</span><span class="o">();</span>
+ <span class="cm">/**
+ * method must be idempotent.
+ */</span>
+ <span class="kt">void</span> <span class="n">prepare</span><span class="o">();</span>
+
+ <span class="cm">/**
+ *
+ * @return a DB connection over which the queries can be executed.
+ */</span>
+ <span class="n">Connection</span> <span class="n">getConnection</span><span class="o">();</span>
+
+ <span class="cm">/**
+ * called once when the system is shutting down, should be idempotent.
+ */</span>
+ <span class="kt">void</span> <span class="n">cleanup</span><span class="o">();</span>
<span class="o">}</span>
</code></pre></div>
<p>Out of the box we support <code>org.apache.storm.jdbc.common.HikariCPConnectionProvider</code> which is an implementation that uses HikariCP.</p>
@@ -127,7 +127,7 @@ in a storm topology.</p>
<p>The main API for inserting data in a table using JDBC is the <code>org.apache.storm.jdbc.mapper.JdbcMapper</code> interface:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">JdbcMapper</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span>
- <span class="n">List</span><span class="o"><</span><span class="n">Column</span><span class="o">></span> <span class="nf">getColumns</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="n">List</span><span class="o"><</span><span class="n">Column</span><span class="o">></span> <span class="n">getColumns</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">);</span>
<span class="o">}</span>
</code></pre></div>
<p>The <code>getColumns()</code> method defines how a storm tuple maps to a list of columns representing a row in a database.
@@ -147,21 +147,21 @@ You can optionally specify a query timeout seconds param that specifies max seco
The default is set to value of topology.message.timeout.secs and a value of -1 will indicate not to set any query timeout.
You should set the query timeout value to be <= topology.message.timeout.secs.</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Map</span> <span class="n">hikariConfigMap</span> <span class="o">=</span> <span class="n">Maps</span><span class="o">.</span><span class="na">newHashMap</span><span class="o">();</span>
-<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSourceClassName"</span><span class="o">,</span><span class="s">"com.mysql.jdbc.jdbc2.optional.MysqlDataSource"</span><span class="o">);</span>
-<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.url"</span><span class="o">,</span> <span class="s">"jdbc:mysql://localhost/test"</span><span class="o">);</span>
-<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.user"</span><span class="o">,</span><span class="s">"root"</span><span class="o">);</span>
-<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.password"</span><span class="o">,</span><span class="s">"password"</span><span class="o">);</span>
-<span class="n">ConnectionProvider</span> <span class="n">connectionProvider</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HikariCPConnectionProvider</span><span class="o">(</span><span class="n">hikariConfigMap</span><span class="o">);</span>
+<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSourceClassName"</span><span class="o">,</span><span class="s">"com.mysql.jdbc.jdbc2.optional.MysqlDataSource"</span><span class="o">);</span>
+<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.url"</span><span class="o">,</span> <span class="s">"jdbc:mysql://localhost/test"</span><span class="o">);</span>
+<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.user"</span><span class="o">,</span><span class="s">"root"</span><span class="o">);</span>
+<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.password"</span><span class="o">,</span><span class="s">"password"</span><span class="o">);</span>
+<span class="n">ConnectionProvider</span> <span class="n">connectionProvider</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HikariCPConnectionProvider</span><span class="o">(</span><span class="n">hikariConfigMap</span><span class="o">);</span>
-<span class="n">String</span> <span class="n">tableName</span> <span class="o">=</span> <span class="s">"user_details"</span><span class="o">;</span>
-<span class="n">JdbcMapper</span> <span class="n">simpleJdbcMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SimpleJdbcMapper</span><span class="o">(</span><span class="n">tableName</span><span class="o">,</span> <span class="n">connectionProvider</span><span class="o">);</span>
+<span class="n">String</span> <span class="n">tableName</span> <span class="o">=</span> <span class="s">"user_details"</span><span class="o">;</span>
+<span class="n">JdbcMapper</span> <span class="n">simpleJdbcMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SimpleJdbcMapper</span><span class="o">(</span><span class="n">tableName</span><span class="o">,</span> <span class="n">connectionProvider</span><span class="o">);</span>
-<span class="n">JdbcInsertBolt</span> <span class="n">userPersistanceBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">JdbcInsertBolt</span><span class="o">(</span><span class="n">connectionProvider</span><span class="o">,</span> <span class="n">simpleJdbcMapper</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withTableName</span><span class="o">(</span><span class="s">"user"</span><span class="o">)</span>
+<span class="n">JdbcInsertBolt</span> <span class="n">userPersistanceBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JdbcInsertBolt</span><span class="o">(</span><span class="n">connectionProvider</span><span class="o">,</span> <span class="n">simpleJdbcMapper</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withTableName</span><span class="o">(</span><span class="s">"user"</span><span class="o">)</span>
<span class="o">.</span><span class="na">withQueryTimeoutSecs</span><span class="o">(</span><span class="mi">30</span><span class="o">);</span>
<span class="n">Or</span>
-<span class="n">JdbcInsertBolt</span> <span class="n">userPersistanceBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">JdbcInsertBolt</span><span class="o">(</span><span class="n">connectionProvider</span><span class="o">,</span> <span class="n">simpleJdbcMapper</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withInsertQuery</span><span class="o">(</span><span class="s">"insert into user values (?,?)"</span><span class="o">)</span>
+<span class="n">JdbcInsertBolt</span> <span class="n">userPersistanceBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JdbcInsertBolt</span><span class="o">(</span><span class="n">connectionProvider</span><span class="o">,</span> <span class="n">simpleJdbcMapper</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withInsertQuery</span><span class="o">(</span><span class="s">"insert into user values (?,?)"</span><span class="o">)</span>
<span class="o">.</span><span class="na">withQueryTimeoutSecs</span><span class="o">(</span><span class="mi">30</span><span class="o">);</span>
</code></pre></div>
<h3 id="simplejdbcmapper">SimpleJdbcMapper</h3>
@@ -181,13 +181,13 @@ automatically figure out the column names and corresponding data types of the ta
Please see <a href="https://github.com/brettwooldridge/HikariCP#configuration-knobs-baby">https://github.com/brettwooldridge/HikariCP#configuration-knobs-baby</a> to learn more about hikari configuration properties.</li>
</ol>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Map</span> <span class="n">hikariConfigMap</span> <span class="o">=</span> <span class="n">Maps</span><span class="o">.</span><span class="na">newHashMap</span><span class="o">();</span>
-<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSourceClassName"</span><span class="o">,</span><span class="s">"com.mysql.jdbc.jdbc2.optional.MysqlDataSource"</span><span class="o">);</span>
-<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.url"</span><span class="o">,</span> <span class="s">"jdbc:mysql://localhost/test"</span><span class="o">);</span>
-<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.user"</span><span class="o">,</span><span class="s">"root"</span><span class="o">);</span>
-<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.password"</span><span class="o">,</span><span class="s">"password"</span><span class="o">);</span>
-<span class="n">ConnectionProvider</span> <span class="n">connectionProvider</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">HikariCPConnectionProvider</span><span class="o">(</span><span class="n">hikariConfigMap</span><span class="o">);</span>
-<span class="n">String</span> <span class="n">tableName</span> <span class="o">=</span> <span class="s">"user_details"</span><span class="o">;</span>
-<span class="n">JdbcMapper</span> <span class="n">simpleJdbcMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SimpleJdbcMapper</span><span class="o">(</span><span class="n">tableName</span><span class="o">,</span> <span class="n">connectionProvider</span><span class="o">);</span>
+<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSourceClassName"</span><span class="o">,</span><span class="s">"com.mysql.jdbc.jdbc2.optional.MysqlDataSource"</span><span class="o">);</span>
+<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.url"</span><span class="o">,</span> <span class="s">"jdbc:mysql://localhost/test"</span><span class="o">);</span>
+<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.user"</span><span class="o">,</span><span class="s">"root"</span><span class="o">);</span>
+<span class="n">hikariConfigMap</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="s">"dataSource.password"</span><span class="o">,</span><span class="s">"password"</span><span class="o">);</span>
+<span class="n">ConnectionProvider</span> <span class="n">connectionProvider</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HikariCPConnectionProvider</span><span class="o">(</span><span class="n">hikariConfigMap</span><span class="o">);</span>
+<span class="n">String</span> <span class="n">tableName</span> <span class="o">=</span> <span class="s">"user_details"</span><span class="o">;</span>
+<span class="n">JdbcMapper</span> <span class="n">simpleJdbcMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SimpleJdbcMapper</span><span class="o">(</span><span class="n">tableName</span><span class="o">,</span> <span class="n">connectionProvider</span><span class="o">);</span>
</code></pre></div>
<p>The mapper initialized in the example above assumes a storm tuple has value for all the columns of the table you intend to insert data into and its <code>getColumn</code>
method will return the columns in the order in which Jdbc connection instance's <code>connection.getMetaData().getColumns();</code> method returns them.</p>
@@ -206,10 +206,10 @@ JdbcMapper simpleJdbcMapper = new SimpleJdbcMapper(columnSchema);
In this table the create_time column has a default value. To ensure only the columns with no default values are inserted
you can initialize the <code>jdbcMapper</code> as below:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">List</span><span class="o"><</span><span class="n">Column</span><span class="o">></span> <span class="n">columnSchema</span> <span class="o">=</span> <span class="n">Lists</span><span class="o">.</span><span class="na">newArrayList</span><span class="o">(</span>
- <span class="k">new</span> <span class="nf">Column</span><span class="o">(</span><span class="s">"user_id"</span><span class="o">,</span> <span class="n">java</span><span class="o">.</span><span class="na">sql</span><span class="o">.</span><span class="na">Types</span><span class="o">.</span><span class="na">INTEGER</span><span class="o">),</span>
- <span class="k">new</span> <span class="nf">Column</span><span class="o">(</span><span class="s">"user_name"</span><span class="o">,</span> <span class="n">java</span><span class="o">.</span><span class="na">sql</span><span class="o">.</span><span class="na">Types</span><span class="o">.</span><span class="na">VARCHAR</span><span class="o">),</span>
- <span class="k">new</span> <span class="nf">Column</span><span class="o">(</span><span class="s">"dept_name"</span><span class="o">,</span> <span class="n">java</span><span class="o">.</span><span class="na">sql</span><span class="o">.</span><span class="na">Types</span><span class="o">.</span><span class="na">VARCHAR</span><span class="o">));</span>
-<span class="n">JdbcMapper</span> <span class="n">simpleJdbcMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SimpleJdbcMapper</span><span class="o">(</span><span class="n">columnSchema</span><span class="o">);</span>
+ <span class="k">new</span> <span class="n">Column</span><span class="o">(</span><span class="s">"user_id"</span><span class="o">,</span> <span class="n">java</span><span class="o">.</span><span class="na">sql</span><span class="o">.</span><span class="na">Types</span><span class="o">.</span><span class="na">INTEGER</span><span class="o">),</span>
+ <span class="k">new</span> <span class="n">Column</span><span class="o">(</span><span class="s">"user_name"</span><span class="o">,</span> <span class="n">java</span><span class="o">.</span><span class="na">sql</span><span class="o">.</span><span class="na">Types</span><span class="o">.</span><span class="na">VARCHAR</span><span class="o">),</span>
+ <span class="k">new</span> <span class="n">Column</span><span class="o">(</span><span class="s">"dept_name"</span><span class="o">,</span> <span class="n">java</span><span class="o">.</span><span class="na">sql</span><span class="o">.</span><span class="na">Types</span><span class="o">.</span><span class="na">VARCHAR</span><span class="o">));</span>
+<span class="n">JdbcMapper</span> <span class="n">simpleJdbcMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SimpleJdbcMapper</span><span class="o">(</span><span class="n">columnSchema</span><span class="o">);</span>
</code></pre></div>
<h3 id="jdbctridentstate">JdbcTridentState</h3>
@@ -219,9 +219,9 @@ See the example below:</p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">JdbcState</span><span class="o">.</span><span class="na">Options</span> <span class="n">options</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JdbcState</span><span class="o">.</span><span class="na">Options</span><span class="o">()</span>
<span class="o">.</span><span class="na">withConnectionProvider</span><span class="o">(</span><span class="n">connectionProvider</span><span class="o">)</span>
<span class="o">.</span><span class="na">withMapper</span><span class="o">(</span><span class="n">jdbcMapper</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withTableName</span><span class="o">(</span><span class="s">"user_details"</span><span class="o">)</span>
+ <span class="o">.</span><span class="na">withTableName</span><span class="o">(</span><span class="s">"user_details"</span><span class="o">)</span>
<span class="o">.</span><span class="na">withQueryTimeoutSecs</span><span class="o">(</span><span class="mi">30</span><span class="o">);</span>
-<span class="n">JdbcStateFactory</span> <span class="n">jdbcStateFactory</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">JdbcStateFactory</span><span class="o">(</span><span class="n">options</span><span class="o">);</span>
+<span class="n">JdbcStateFactory</span> <span class="n">jdbcStateFactory</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JdbcStateFactory</span><span class="o">(</span><span class="n">options</span><span class="o">);</span>
</code></pre></div>
<p>similar to <code>JdbcInsertBolt</code> you can specify a custom insert query using <code>withInsertQuery</code> instead of specifying a table name.</p>
@@ -229,9 +229,9 @@ See the example below:</p>
<p>We support <code>select</code> queries from databases to allow enrichment of storm tuples in a topology. The main API for
executing select queries against a database using JDBC is the <code>org.apache.storm.jdbc.mapper.JdbcLookupMapper</code> interface:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">);</span>
- <span class="n">List</span><span class="o"><</span><span class="n">Column</span><span class="o">></span> <span class="nf">getColumns</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">);</span>
- <span class="n">List</span><span class="o"><</span><span class="n">Values</span><span class="o">></span> <span class="nf">toTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">input</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">Column</span><span class="o">></span> <span class="n">columns</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="p">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">);</span>
+ <span class="n">List</span><span class="o"><</span><span class="n">Column</span><span class="o">></span> <span class="n">getColumns</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="n">List</span><span class="o"><</span><span class="n">Values</span><span class="o">></span> <span class="n">toTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">input</span><span class="o">,</span> <span class="n">List</span><span class="o"><</span><span class="n">Column</span><span class="o">></span> <span class="n">columns</span><span class="o">);</span>
</code></pre></div>
<p>The <code>declareOutputFields</code> method is used to indicate what fields will be emitted as part of output tuple of processing a storm
tuple. </p>
@@ -265,18 +265,18 @@ So in the example below if the input tuple had fields <code>user_id, create_date
will return the value of <code>tuple.getValueByField("user_id")</code> which will be used as the value in <code>?</code> of select query.
For each output row from DB, <code>SimpleJdbcLookupMapper.toTuple()</code> will use the <code>user_id, create_date</code> from the input tuple as
is adding only <code>user_name</code> from the resulting row and returning these 3 fields as a single output tuple.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Fields</span> <span class="n">outputFields</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"user_id"</span><span class="o">,</span> <span class="s">"user_name"</span><span class="o">,</span> <span class="s">"create_date"</span><span class="o">);</span>
-<span class="n">List</span><span class="o"><</span><span class="n">Column</span><span class="o">></span> <span class="n">queryParamColumns</span> <span class="o">=</span> <span class="n">Lists</span><span class="o">.</span><span class="na">newArrayList</span><span class="o">(</span><span class="k">new</span> <span class="nf">Column</span><span class="o">(</span><span class="s">"user_id"</span><span class="o">,</span> <span class="n">Types</span><span class="o">.</span><span class="na">INTEGER</span><span class="o">));</span>
-<span class="k">this</span><span class="o">.</span><span class="na">jdbcLookupMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SimpleJdbcLookupMapper</span><span class="o">(</span><span class="n">outputFields</span><span class="o">,</span> <span class="n">queryParamColumns</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">Fields</span> <span class="n">outputFields</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"user_id"</span><span class="o">,</span> <span class="s">"user_name"</span><span class="o">,</span> <span class="s">"create_date"</span><span class="o">);</span>
+<span class="n">List</span><span class="o"><</span><span class="n">Column</span><span class="o">></span> <span class="n">queryParamColumns</span> <span class="o">=</span> <span class="n">Lists</span><span class="o">.</span><span class="na">newArrayList</span><span class="o">(</span><span class="k">new</span> <span class="n">Column</span><span class="o">(</span><span class="s">"user_id"</span><span class="o">,</span> <span class="n">Types</span><span class="o">.</span><span class="na">INTEGER</span><span class="o">));</span>
+<span class="k">this</span><span class="o">.</span><span class="na">jdbcLookupMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SimpleJdbcLookupMapper</span><span class="o">(</span><span class="n">outputFields</span><span class="o">,</span> <span class="n">queryParamColumns</span><span class="o">);</span>
</code></pre></div>
<h3 id="jdbclookupbolt">JdbcLookupBolt</h3>
<p>To use the <code>JdbcLookupBolt</code>, construct an instance of it using a <code>ConnectionProvider</code> instance, <code>JdbcLookupMapper</code> instance and the select query to execute.
You can optionally specify a query timeout seconds param that specifies max seconds the select query can take.
The default is set to value of topology.message.timeout.secs. You should set this value to be <= topology.message.timeout.secs.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">String</span> <span class="n">selectSql</span> <span class="o">=</span> <span class="s">"select user_name from user_details where user_id = ?"</span><span class="o">;</span>
-<span class="n">SimpleJdbcLookupMapper</span> <span class="n">lookupMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SimpleJdbcLookupMapper</span><span class="o">(</span><span class="n">outputFields</span><span class="o">,</span> <span class="n">queryParamColumns</span><span class="o">)</span>
-<span class="n">JdbcLookupBolt</span> <span class="n">userNameLookupBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">JdbcLookupBolt</span><span class="o">(</span><span class="n">connectionProvider</span><span class="o">,</span> <span class="n">selectSql</span><span class="o">,</span> <span class="n">lookupMapper</span><span class="o">)</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">String</span> <span class="n">selectSql</span> <span class="o">=</span> <span class="s">"select user_name from user_details where user_id = ?"</span><span class="o">;</span>
+<span class="n">SimpleJdbcLookupMapper</span> <span class="n">lookupMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SimpleJdbcLookupMapper</span><span class="o">(</span><span class="n">outputFields</span><span class="o">,</span> <span class="n">queryParamColumns</span><span class="o">)</span>
+<span class="n">JdbcLookupBolt</span> <span class="n">userNameLookupBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JdbcLookupBolt</span><span class="o">(</span><span class="n">connectionProvider</span><span class="o">,</span> <span class="n">selectSql</span><span class="o">,</span> <span class="n">lookupMapper</span><span class="o">)</span>
<span class="o">.</span><span class="na">withQueryTimeoutSecs</span><span class="o">(</span><span class="mi">30</span><span class="o">);</span>
</code></pre></div>
<h3 id="jdbctridentstate-for-lookup">JdbcTridentState for lookup</h3>
@@ -284,11 +284,11 @@ The default is set to value of topology.message.timeout.secs. You should set thi
<p>We also support a trident query state that can be used with trident topologies. </p>
<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">JdbcState</span><span class="o">.</span><span class="na">Options</span> <span class="n">options</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JdbcState</span><span class="o">.</span><span class="na">Options</span><span class="o">()</span>
<span class="o">.</span><span class="na">withConnectionProvider</span><span class="o">(</span><span class="n">connectionProvider</span><span class="o">)</span>
- <span class="o">.</span><span class="na">withJdbcLookupMapper</span><span class="o">(</span><span class="k">new</span> <span class="nf">SimpleJdbcLookupMapper</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"user_name"</span><span class="o">),</span> <span class="n">Lists</span><span class="o">.</span><span class="na">newArrayList</span><span class="o">(</span><span class="k">new</span> <span class="nf">Column</span><span class="o">(</span><span class="s">"user_id"</span><span class="o">,</span> <span class="n">Types</span><span class="o">.</span><span class="na">INTEGER</span><span class="o">))))</span>
- <span class="o">.</span><span class="na">withSelectQuery</span><span class="o">(</span><span class="s">"select user_name from user_details where user_id = ?"</span><span class="o">);</span>
+ <span class="o">.</span><span class="na">withJdbcLookupMapper</span><span class="o">(</span><span class="k">new</span> <span class="n">SimpleJdbcLookupMapper</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"user_name"</span><span class="o">),</span> <span class="n">Lists</span><span class="o">.</span><span class="na">newArrayList</span><span class="o">(</span><span class="k">new</span> <span class="n">Column</span><span class="o">(</span><span class="s">"user_id"</span><span class="o">,</span> <span class="n">Types</span><span class="o">.</span><span class="na">INTEGER</span><span class="o">))))</span>
+ <span class="o">.</span><span class="na">withSelectQuery</span><span class="o">(</span><span class="s">"select user_name from user_details where user_id = ?"</span><span class="o">);</span>
<span class="o">.</span><span class="na">withQueryTimeoutSecs</span><span class="o">(</span><span class="mi">30</span><span class="o">);</span>
</code></pre></div>
-<h2 id="example:">Example:</h2>
+<h2 id="example">Example:</h2>
<p>A runnable example can be found in the <code>src/test/java/topology</code> directory.</p>
@@ -318,7 +318,7 @@ storm jar org.apache.storm.jdbc.topology.UserPersistanceTopology <dataSourceClas
</ul>
<p>To make it work with Mysql, you can add the following to the pom.xml</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"><dependency>
+<div class="highlight"><pre><code class="language-" data-lang=""><dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.31</version>
@@ -326,7 +326,7 @@ storm jar org.apache.storm.jdbc.topology.UserPersistanceTopology <dataSourceClas
</code></pre></div>
<p>You can generate a single jar with dependencies using mvn assembly plugin. To use the plugin add the following to your pom.xml and execute
<code>mvn clean compile assembly:single</code></p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"><plugin>
+<div class="highlight"><pre><code class="language-" data-lang=""><plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<archive>
@@ -346,7 +346,7 @@ storm jar ~/repo/incubator-storm/external/storm-jdbc/target/storm-jdbc-0.10.0-SN
</code></p>
<p>You can execute a select query against the user table which should show newly inserted rows:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">select * from user;
+<div class="highlight"><pre><code class="language-" data-lang="">select * from user;
</code></pre></div>
<p>For trident you can view <code>org.apache.storm.jdbc.topology.UserPersistanceTridentTopology</code>.</p>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/storm-kafka.html
----------------------------------------------------------------------
diff --git a/_site/documentation/storm-kafka.html b/_site/documentation/storm-kafka.html
index 1eb9f45..3fa9124 100644
--- a/_site/documentation/storm-kafka.html
+++ b/_site/documentation/storm-kafka.html
@@ -120,14 +120,14 @@ should set host.refreshFreqSecs to your chosen value.</p>
<p>This is an alternative implementation where broker -> partition information is static. In order to construct an instance
of this class, you need to first construct an instance of GlobalPartitionInformation.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">Broker</span> <span class="n">brokerForPartition0</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Broker</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">);</span><span class="c1">//localhost:9092</span>
- <span class="n">Broker</span> <span class="n">brokerForPartition1</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Broker</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">,</span> <span class="mi">9092</span><span class="o">);</span><span class="c1">//localhost:9092 but we specified the port explicitly</span>
- <span class="n">Broker</span> <span class="n">brokerForPartition2</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Broker</span><span class="o">(</span><span class="s">"localhost:9092"</span><span class="o">);</span><span class="c1">//localhost:9092 specified as one string.</span>
- <span class="n">GlobalPartitionInformation</span> <span class="n">partitionInfo</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">GlobalPartitionInformation</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">Broker</span> <span class="n">brokerForPartition0</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Broker</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">);</span><span class="c1">//localhost:9092</span>
+ <span class="n">Broker</span> <span class="n">brokerForPartition1</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Broker</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">,</span> <span class="mi">9092</span><span class="o">);</span><span class="c1">//localhost:9092 but we specified the port explicitly</span>
+ <span class="n">Broker</span> <span class="n">brokerForPartition2</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Broker</span><span class="o">(</span><span class="s">"localhost:9092"</span><span class="o">);</span><span class="c1">//localhost:9092 specified as one string.</span>
+ <span class="n">GlobalPartitionInformation</span> <span class="n">partitionInfo</span> <span class="o">=</span> <span class="k">new</span> <span class="n">GlobalPartitionInformation</span><span class="o">();</span>
<span class="n">partitionInfo</span><span class="o">.</span><span class="na">addPartition</span><span class="o">(</span><span class="mi">0</span><span class="o">,</span> <span class="n">brokerForPartition0</span><span class="o">);</span><span class="c1">//mapping from partition 0 to brokerForPartition0</span>
<span class="n">partitionInfo</span><span class="o">.</span><span class="na">addPartition</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="n">brokerForPartition1</span><span class="o">);</span><span class="c1">//mapping from partition 1 to brokerForPartition1</span>
<span class="n">partitionInfo</span><span class="o">.</span><span class="na">addPartition</span><span class="o">(</span><span class="mi">2</span><span class="o">,</span> <span class="n">brokerForPartition2</span><span class="o">);</span><span class="c1">//mapping from partition 2 to brokerForPartition2</span>
- <span class="n">StaticHosts</span> <span class="n">hosts</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">StaticHosts</span><span class="o">(</span><span class="n">partitionInfo</span><span class="o">);</span>
+ <span class="n">StaticHosts</span> <span class="n">hosts</span> <span class="o">=</span> <span class="k">new</span> <span class="n">StaticHosts</span><span class="o">(</span><span class="n">partitionInfo</span><span class="o">);</span>
</code></pre></div>
<h3 id="kafkaconfig">KafkaConfig</h3>
@@ -153,7 +153,7 @@ In addition to these parameters, SpoutConfig contains the following fields that
```java
// setting for how often to save the current Kafka offset to ZooKeeper
public long stateUpdateIntervalMs = 2000;</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">// Exponential back-off retry settings. These are used when retrying messages after a bolt
+<div class="highlight"><pre><code class="language-" data-lang="">// Exponential back-off retry settings. These are used when retrying messages after a bolt
// calls OutputCollector.fail().
// Note: be sure to set backtype.storm.Config.MESSAGE_TIMEOUT_SECS appropriately to prevent
// resubmitting the message while still retrying.
@@ -163,12 +163,12 @@ public long retryDelayMaxMs = 60 * 1000;
// if set to true, spout will set Kafka topic as the emitted Stream ID
public boolean topicAsStreamId = false;
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">Core KafkaSpout only accepts an instance of SpoutConfig.
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">Core KafkaSpout only accepts an instance of SpoutConfig.
TridentKafkaConfig is another extension of KafkaConfig.
TridentKafkaEmitter only accepts TridentKafkaConfig.
-The KafkaConfig class also has bunch of public variables that controls your application's behavior. Here are defaults:
+The KafkaConfig class also has bunch of public variables that controls your application's behavior. Here are defaults:
```java
public int fetchSizeBytes = 1024 * 1024;
public int socketTimeoutMs = 10000;
@@ -187,8 +187,8 @@ The KafkaConfig class also has bunch of public variables that controls your appl
<p>MultiScheme is an interface that dictates how the byte[] consumed from Kafka gets transformed into a storm tuple. It
also controls the naming of your output field.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="kd">public</span> <span class="n">Iterable</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="nf">deserialize</span><span class="o">(</span><span class="kt">byte</span><span class="o">[]</span> <span class="n">ser</span><span class="o">);</span>
- <span class="kd">public</span> <span class="n">Fields</span> <span class="nf">getOutputFields</span><span class="o">();</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="kd">public</span> <span class="n">Iterable</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Object</span><span class="o">>></span> <span class="n">deserialize</span><span class="o">(</span><span class="kt">byte</span><span class="o">[]</span> <span class="n">ser</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="n">Fields</span> <span class="n">getOutputFields</span><span class="o">();</span>
</code></pre></div>
<p>The default <code>RawMultiScheme</code> just takes the <code>byte[]</code> and returns a tuple with <code>byte[]</code> as is. The name of the
outputField is "bytes". There are alternative implementation like <code>SchemeAsMultiScheme</code> and
@@ -197,17 +197,17 @@ outputField is "bytes". There are alternative implementation like <co
<h3 id="examples">Examples</h3>
<h4 id="core-spout">Core Spout</h4>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">BrokerHosts</span> <span class="n">hosts</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">ZkHosts</span><span class="o">(</span><span class="n">zkConnString</span><span class="o">);</span>
-<span class="n">SpoutConfig</span> <span class="n">spoutConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SpoutConfig</span><span class="o">(</span><span class="n">hosts</span><span class="o">,</span> <span class="n">topicName</span><span class="o">,</span> <span class="s">"/"</span> <span class="o">+</span> <span class="n">topicName</span><span class="o">,</span> <span class="n">UUID</span><span class="o">.</span><span class="na">randomUUID</span><span class="o">().</span><span class="na">toString</span><span class="o">());</span>
-<span class="n">spoutConfig</span><span class="o">.</span><span class="na">scheme</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SchemeAsMultiScheme</span><span class="o">(</span><span class="k">new</span> <span class="nf">StringScheme</span><span class="o">());</span>
-<span class="n">KafkaSpout</span> <span class="n">kafkaSpout</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">KafkaSpout</span><span class="o">(</span><span class="n">spoutConfig</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">BrokerHosts</span> <span class="n">hosts</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ZkHosts</span><span class="o">(</span><span class="n">zkConnString</span><span class="o">);</span>
+<span class="n">SpoutConfig</span> <span class="n">spoutConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SpoutConfig</span><span class="o">(</span><span class="n">hosts</span><span class="o">,</span> <span class="n">topicName</span><span class="o">,</span> <span class="s">"/"</span> <span class="o">+</span> <span class="n">topicName</span><span class="o">,</span> <span class="n">UUID</span><span class="o">.</span><span class="na">randomUUID</span><span class="o">().</span><span class="na">toString</span><span class="o">());</span>
+<span class="n">spoutConfig</span><span class="o">.</span><span class="na">scheme</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SchemeAsMultiScheme</span><span class="o">(</span><span class="k">new</span> <span class="n">StringScheme</span><span class="o">());</span>
+<span class="n">KafkaSpout</span> <span class="n">kafkaSpout</span> <span class="o">=</span> <span class="k">new</span> <span class="n">KafkaSpout</span><span class="o">(</span><span class="n">spoutConfig</span><span class="o">);</span>
</code></pre></div>
<h4 id="trident-spout">Trident Spout</h4>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TridentTopology</span><span class="o">();</span>
-<span class="n">BrokerHosts</span> <span class="n">zk</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">ZkHosts</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">);</span>
-<span class="n">TridentKafkaConfig</span> <span class="n">spoutConf</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">TridentKafkaConfig</span><span class="o">(</span><span class="n">zk</span><span class="o">,</span> <span class="s">"test-topic"</span><span class="o">);</span>
-<span class="n">spoutConf</span><span class="o">.</span><span class="na">scheme</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">SchemeAsMultiScheme</span><span class="o">(</span><span class="k">new</span> <span class="nf">StringScheme</span><span class="o">());</span>
-<span class="n">OpaqueTridentKafkaSpout</span> <span class="n">spout</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">OpaqueTridentKafkaSpout</span><span class="o">(</span><span class="n">spoutConf</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TridentTopology</span> <span class="n">topology</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TridentTopology</span><span class="o">();</span>
+<span class="n">BrokerHosts</span> <span class="n">zk</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ZkHosts</span><span class="o">(</span><span class="s">"localhost"</span><span class="o">);</span>
+<span class="n">TridentKafkaConfig</span> <span class="n">spoutConf</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TridentKafkaConfig</span><span class="o">(</span><span class="n">zk</span><span class="o">,</span> <span class="s">"test-topic"</span><span class="o">);</span>
+<span class="n">spoutConf</span><span class="o">.</span><span class="na">scheme</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SchemeAsMultiScheme</span><span class="o">(</span><span class="k">new</span> <span class="n">StringScheme</span><span class="o">());</span>
+<span class="n">OpaqueTridentKafkaSpout</span> <span class="n">spout</span> <span class="o">=</span> <span class="k">new</span> <span class="n">OpaqueTridentKafkaSpout</span><span class="o">(</span><span class="n">spoutConf</span><span class="o">);</span>
</code></pre></div>
<h3 id="how-kafkaspout-stores-offsets-of-a-kafka-topic-and-recovers-in-case-of-failures">How KafkaSpout stores offsets of a Kafka topic and recovers in case of failures</h3>
@@ -274,8 +274,8 @@ storm.kafka.trident.TridentKafkaUpdater.</p>
<h3 id="tupletokafkamapper-and-tridenttupletokafkamapper">TupleToKafkaMapper and TridentTupleToKafkaMapper</h3>
<p>These interfaces have 2 methods defined:</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">K</span> <span class="nf">getKeyFromTuple</span><span class="o">(</span><span class="n">Tuple</span><span class="o">/</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">);</span>
- <span class="n">V</span> <span class="nf">getMessageFromTuple</span><span class="o">(</span><span class="n">Tuple</span><span class="o">/</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">);</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="n">K</span> <span class="nf">getKeyFromTuple</span><span class="p">(</span><span class="n">Tuple</span><span class="o">/</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">);</span>
+ <span class="n">V</span> <span class="n">getMessageFromTuple</span><span class="o">(</span><span class="n">Tuple</span><span class="o">/</span><span class="n">TridentTuple</span> <span class="n">tuple</span><span class="o">);</span>
</code></pre></div>
<p>As the name suggests, these methods are called to map a tuple to Kafka key and Kafka message. If you just want one field
as key and one field as value, then you can use the provided FieldNameBasedTupleToKafkaMapper.java
@@ -308,57 +308,58 @@ map with key kafka.broker.properties.</p>
<p>For the bolt :
```java
TopologyBuilder builder = new TopologyBuilder();</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"> Fields fields = new Fields("key", "message");
+<div class="highlight"><pre><code class="language-" data-lang=""> Fields fields = new Fields("key", "message");
FixedBatchSpout spout = new FixedBatchSpout(fields, 4,
- new Values("storm", "1"),
- new Values("trident", "1"),
- new Values("needs", "1"),
- new Values("javadoc", "1")
+ new Values("storm", "1"),
+ new Values("trident", "1"),
+ new Values("needs", "1"),
+ new Values("javadoc", "1")
);
spout.setCycle(true);
- builder.setSpout("spout", spout, 5);
+ builder.setSpout("spout", spout, 5);
KafkaBolt bolt = new KafkaBolt()
- .withTopicSelector(new DefaultTopicSelector("test"))
+ .withTopicSelector(new DefaultTopicSelector("test"))
.withTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper());
- builder.setBolt("forwardToKafka", bolt, 8).shuffleGrouping("spout");
+ builder.setBolt("forwardToKafka", bolt, 8).shuffleGrouping("spout");
Config conf = new Config();
//set producer properties.
Properties props = new Properties();
- props.put("metadata.broker.list", "localhost:9092");
- props.put("request.required.acks", "1");
- props.put("serializer.class", "kafka.serializer.StringEncoder");
+ props.put("metadata.broker.list", "localhost:9092");
+ props.put("request.required.acks", "1");
+ props.put("serializer.class", "kafka.serializer.StringEncoder");
conf.put(KafkaBolt.KAFKA_BROKER_PROPERTIES, props);
- StormSubmitter.submitTopology("kafkaboltTest", conf, builder.createTopology());
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">For Trident:
+ StormSubmitter.submitTopology("kafkaboltTest", conf, builder.createTopology());
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">
+For Trident:
```java
- Fields fields = new Fields("word", "count");
+ Fields fields = new Fields("word", "count");
FixedBatchSpout spout = new FixedBatchSpout(fields, 4,
- new Values("storm", "1"),
- new Values("trident", "1"),
- new Values("needs", "1"),
- new Values("javadoc", "1")
+ new Values("storm", "1"),
+ new Values("trident", "1"),
+ new Values("needs", "1"),
+ new Values("javadoc", "1")
);
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
- Stream stream = topology.newStream("spout1", spout);
+ Stream stream = topology.newStream("spout1", spout);
TridentKafkaStateFactory stateFactory = new TridentKafkaStateFactory()
- .withKafkaTopicSelector(new DefaultTopicSelector("test"))
- .withTridentTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper("word", "count"));
+ .withKafkaTopicSelector(new DefaultTopicSelector("test"))
+ .withTridentTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper("word", "count"));
stream.partitionPersist(stateFactory, fields, new TridentKafkaUpdater(), new Fields());
Config conf = new Config();
//set producer properties.
Properties props = new Properties();
- props.put("metadata.broker.list", "localhost:9092");
- props.put("request.required.acks", "1");
- props.put("serializer.class", "kafka.serializer.StringEncoder");
+ props.put("metadata.broker.list", "localhost:9092");
+ props.put("request.required.acks", "1");
+ props.put("serializer.class", "kafka.serializer.StringEncoder");
conf.put(TridentKafkaState.KAFKA_BROKER_PROPERTIES, props);
- StormSubmitter.submitTopology("kafkaTridentTest", conf, topology.build());
+ StormSubmitter.submitTopology("kafkaTridentTest", conf, topology.build());
</code></pre></div>
http://git-wip-us.apache.org/repos/asf/storm/blob/dfa66d7d/_site/documentation/storm-redis.html
----------------------------------------------------------------------
diff --git a/_site/documentation/storm-redis.html b/_site/documentation/storm-redis.html
index e21f046..4a2dc97 100644
--- a/_site/documentation/storm-redis.html
+++ b/_site/documentation/storm-redis.html
@@ -96,7 +96,7 @@
<h2 id="usage">Usage</h2>
-<h3 id="how-do-i-use-it?">How do I use it?</h3>
+<h3 id="how-do-i-use-it">How do I use it?</h3>
<p>use it as a maven dependency:</p>
<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><dependency></span>
@@ -117,111 +117,117 @@
<p>These interfaces are combined with <code>RedisLookupMapper</code> and <code>RedisStoreMapper</code> which fit <code>RedisLookupBolt</code> and <code>RedisStoreBolt</code> respectively.</p>
<h4 id="redislookupbolt-example">RedisLookupBolt example</h4>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">class</span> <span class="nc">WordCountRedisLookupMapper</span> <span class="kd">implements</span> <span class="n">RedisLookupMapper</span> <span class="o">{</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java">
+<span class="kd">class</span> <span class="nc">WordCountRedisLookupMapper</span> <span class="kd">implements</span> <span class="n">RedisLookupMapper</span> <span class="o">{</span>
<span class="kd">private</span> <span class="n">RedisDataTypeDescription</span> <span class="n">description</span><span class="o">;</span>
- <span class="kd">private</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">hashKey</span> <span class="o">=</span> <span class="s">"wordCount"</span><span class="o">;</span>
+ <span class="kd">private</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">hashKey</span> <span class="o">=</span> <span class="s">"wordCount"</span><span class="o">;</span>
- <span class="kd">public</span> <span class="nf">WordCountRedisLookupMapper</span><span class="o">()</span> <span class="o">{</span>
- <span class="n">description</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">RedisDataTypeDescription</span><span class="o">(</span>
+ <span class="kd">public</span> <span class="n">WordCountRedisLookupMapper</span><span class="o">()</span> <span class="o">{</span>
+ <span class="n">description</span> <span class="o">=</span> <span class="k">new</span> <span class="n">RedisDataTypeDescription</span><span class="o">(</span>
<span class="n">RedisDataTypeDescription</span><span class="o">.</span><span class="na">RedisDataType</span><span class="o">.</span><span class="na">HASH</span><span class="o">,</span> <span class="n">hashKey</span><span class="o">);</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">Values</span><span class="o">></span> <span class="nf">toTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">input</span><span class="o">,</span> <span class="n">Object</span> <span class="n">value</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">List</span><span class="o"><</span><span class="n">Values</span><span class="o">></span> <span class="n">toTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">input</span><span class="o">,</span> <span class="n">Object</span> <span class="n">value</span><span class="o">)</span> <span class="o">{</span>
<span class="n">String</span> <span class="n">member</span> <span class="o">=</span> <span class="n">getKeyFromTuple</span><span class="o">(</span><span class="n">input</span><span class="o">);</span>
<span class="n">List</span><span class="o"><</span><span class="n">Values</span><span class="o">></span> <span class="n">values</span> <span class="o">=</span> <span class="n">Lists</span><span class="o">.</span><span class="na">newArrayList</span><span class="o">();</span>
- <span class="n">values</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">member</span><span class="o">,</span> <span class="n">value</span><span class="o">));</span>
+ <span class="n">values</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">member</span><span class="o">,</span> <span class="n">value</span><span class="o">));</span>
<span class="k">return</span> <span class="n">values</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"wordName"</span><span class="o">,</span> <span class="s">"count"</span><span class="o">));</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"wordName"</span><span class="o">,</span> <span class="s">"count"</span><span class="o">));</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="n">RedisDataTypeDescription</span> <span class="nf">getDataTypeDescription</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">RedisDataTypeDescription</span> <span class="n">getDataTypeDescription</span><span class="o">()</span> <span class="o">{</span>
<span class="k">return</span> <span class="n">description</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="n">String</span> <span class="nf">getKeyFromTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
- <span class="k">return</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getStringByField</span><span class="o">(</span><span class="s">"word"</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="n">String</span> <span class="n">getKeyFromTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="k">return</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getStringByField</span><span class="o">(</span><span class="s">"word"</span><span class="o">);</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="n">String</span> <span class="nf">getValueFromTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">String</span> <span class="n">getValueFromTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
<span class="k">return</span> <span class="kc">null</span><span class="o">;</span>
<span class="o">}</span>
<span class="o">}</span>
-</code></pre></div><div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">JedisPoolConfig</span> <span class="n">poolConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JedisPoolConfig</span><span class="o">.</span><span class="na">Builder</span><span class="o">()</span>
+
+</code></pre></div><div class="highlight"><pre><code class="language-java" data-lang="java">
+<span class="n">JedisPoolConfig</span> <span class="n">poolConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JedisPoolConfig</span><span class="o">.</span><span class="na">Builder</span><span class="o">()</span>
<span class="o">.</span><span class="na">setHost</span><span class="o">(</span><span class="n">host</span><span class="o">).</span><span class="na">setPort</span><span class="o">(</span><span class="n">port</span><span class="o">).</span><span class="na">build</span><span class="o">();</span>
-<span class="n">RedisLookupMapper</span> <span class="n">lookupMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">WordCountRedisLookupMapper</span><span class="o">();</span>
-<span class="n">RedisLookupBolt</span> <span class="n">lookupBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">RedisLookupBolt</span><span class="o">(</span><span class="n">poolConfig</span><span class="o">,</span> <span class="n">lookupMapper</span><span class="o">);</span>
+<span class="n">RedisLookupMapper</span> <span class="n">lookupMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">WordCountRedisLookupMapper</span><span class="o">();</span>
+<span class="n">RedisLookupBolt</span> <span class="n">lookupBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">RedisLookupBolt</span><span class="o">(</span><span class="n">poolConfig</span><span class="o">,</span> <span class="n">lookupMapper</span><span class="o">);</span>
</code></pre></div>
<h4 id="redisstorebolt-example">RedisStoreBolt example</h4>
-<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">class</span> <span class="nc">WordCountStoreMapper</span> <span class="kd">implements</span> <span class="n">RedisStoreMapper</span> <span class="o">{</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java">
+<span class="kd">class</span> <span class="nc">WordCountStoreMapper</span> <span class="kd">implements</span> <span class="n">RedisStoreMapper</span> <span class="o">{</span>
<span class="kd">private</span> <span class="n">RedisDataTypeDescription</span> <span class="n">description</span><span class="o">;</span>
- <span class="kd">private</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">hashKey</span> <span class="o">=</span> <span class="s">"wordCount"</span><span class="o">;</span>
+ <span class="kd">private</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">hashKey</span> <span class="o">=</span> <span class="s">"wordCount"</span><span class="o">;</span>
- <span class="kd">public</span> <span class="nf">WordCountStoreMapper</span><span class="o">()</span> <span class="o">{</span>
- <span class="n">description</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">RedisDataTypeDescription</span><span class="o">(</span>
+ <span class="kd">public</span> <span class="n">WordCountStoreMapper</span><span class="o">()</span> <span class="o">{</span>
+ <span class="n">description</span> <span class="o">=</span> <span class="k">new</span> <span class="n">RedisDataTypeDescription</span><span class="o">(</span>
<span class="n">RedisDataTypeDescription</span><span class="o">.</span><span class="na">RedisDataType</span><span class="o">.</span><span class="na">HASH</span><span class="o">,</span> <span class="n">hashKey</span><span class="o">);</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="n">RedisDataTypeDescription</span> <span class="nf">getDataTypeDescription</span><span class="o">()</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">RedisDataTypeDescription</span> <span class="n">getDataTypeDescription</span><span class="o">()</span> <span class="o">{</span>
<span class="k">return</span> <span class="n">description</span><span class="o">;</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="n">String</span> <span class="nf">getKeyFromTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
- <span class="k">return</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getStringByField</span><span class="o">(</span><span class="s">"word"</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="n">String</span> <span class="n">getKeyFromTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="k">return</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getStringByField</span><span class="o">(</span><span class="s">"word"</span><span class="o">);</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="n">String</span> <span class="nf">getValueFromTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
- <span class="k">return</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getStringByField</span><span class="o">(</span><span class="s">"count"</span><span class="o">);</span>
+ <span class="kd">public</span> <span class="n">String</span> <span class="n">getValueFromTuple</span><span class="o">(</span><span class="n">ITuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span>
+ <span class="k">return</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getStringByField</span><span class="o">(</span><span class="s">"count"</span><span class="o">);</span>
<span class="o">}</span>
<span class="o">}</span>
-</code></pre></div><div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">JedisPoolConfig</span> <span class="n">poolConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JedisPoolConfig</span><span class="o">.</span><span class="na">Builder</span><span class="o">()</span>
+</code></pre></div><div class="highlight"><pre><code class="language-java" data-lang="java">
+<span class="n">JedisPoolConfig</span> <span class="n">poolConfig</span> <span class="o">=</span> <span class="k">new</span> <span class="n">JedisPoolConfig</span><span class="o">.</span><span class="na">Builder</span><span class="o">()</span>
<span class="o">.</span><span class="na">setHost</span><span class="o">(</span><span class="n">host</span><span class="o">).</span><span class="na">setPort</span><span class="o">(</span><span class="n">port</span><span class="o">).</span><span class="na">build</span><span class="o">();</span>
-<span class="n">RedisStoreMapper</span> <span class="n">storeMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">WordCountStoreMapper</span><span class="o">();</span>
-<span class="n">RedisStoreBolt</span> <span class="n">storeBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">RedisStoreBolt</span><span class="o">(</span><span class="n">poolConfig</span><span class="o">,</span> <span class="n">storeMapper</span><span class="o">);</span>
+<span class="n">RedisStoreMapper</span> <span class="n">storeMapper</span> <span class="o">=</span> <span class="k">new</span> <span class="n">WordCountStoreMapper</span><span class="o">();</span>
+<span class="n">RedisStoreBolt</span> <span class="n">storeBolt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">RedisStoreBolt</span><span class="o">(</span><span class="n">poolConfig</span><span class="o">,</span> <span class="n">storeMapper</span><span class="o">);</span>
</code></pre></div>
<h3 id="for-non-simple-bolt">For non-simple Bolt</h3>
<p>If your scenario doesn't fit <code>RedisStoreBolt</code> and <code>RedisLookupBolt</code>, storm-redis also provides <code>AbstractRedisBolt</code> to let you extend and apply your business logic.</p>
-<div class="highlight"><pre><code class="language-java" data-lang="java"> <span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">LookupWordTotalCountBolt</span> <span class="kd">extends</span> <span class="n">AbstractRedisBolt</span> <span class="o">{</span>
+<div class="highlight"><pre><code class="language-java" data-lang="java">
+ <span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">LookupWordTotalCountBolt</span> <span class="kd">extends</span> <span class="n">AbstractRedisBolt</span> <span class="o">{</span>
<span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">Logger</span> <span class="n">LOG</span> <span class="o">=</span> <span class="n">LoggerFactory</span><span class="o">.</span><span class="na">getLogger</span><span class="o">(</span><span class="n">LookupWordTotalCountBolt</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
- <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">Random</span> <span class="n">RANDOM</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Random</span><span class="o">();</span>
+ <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">Random</span> <span class="n">RANDOM</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Random</span><span class="o">();</span>
- <span class="kd">public</span> <span class="nf">LookupWordTotalCountBolt</span><span class="o">(</span><span class="n">JedisPoolConfig</span> <span class="n">config</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">LookupWordTotalCountBolt</span><span class="o">(</span><span class="n">JedisPoolConfig</span> <span class="n">config</span><span class="o">)</span> <span class="o">{</span>
<span class="kd">super</span><span class="o">(</span><span class="n">config</span><span class="o">);</span>
<span class="o">}</span>
- <span class="kd">public</span> <span class="nf">LookupWordTotalCountBolt</span><span class="o">(</span><span class="n">JedisClusterConfig</span> <span class="n">config</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="n">LookupWordTotalCountBolt</span><span class="o">(</span><span class="n">JedisClusterConfig</span> <span class="n">config</span><span class="o">)</span> <span class="o">{</span>
<span class="kd">super</span><span class="o">(</span><span class="n">config</span><span class="o">);</span>
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">input</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">input</span><span class="o">)</span> <span class="o">{</span>
<span class="n">JedisCommands</span> <span class="n">jedisCommands</span> <span class="o">=</span> <span class="kc">null</span><span class="o">;</span>
<span class="k">try</span> <span class="o">{</span>
<span class="n">jedisCommands</span> <span class="o">=</span> <span class="n">getInstance</span><span class="o">();</span>
- <span class="n">String</span> <span class="n">wordName</span> <span class="o">=</span> <span class="n">input</span><span class="o">.</span><span class="na">getStringByField</span><span class="o">(</span><span class="s">"word"</span><span class="o">);</span>
+ <span class="n">String</span> <span class="n">wordName</span> <span class="o">=</span> <span class="n">input</span><span class="o">.</span><span class="na">getStringByField</span><span class="o">(</span><span class="s">"word"</span><span class="o">);</span>
<span class="n">String</span> <span class="n">countStr</span> <span class="o">=</span> <span class="n">jedisCommands</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="n">wordName</span><span class="o">);</span>
<span class="k">if</span> <span class="o">(</span><span class="n">countStr</span> <span class="o">!=</span> <span class="kc">null</span><span class="o">)</span> <span class="o">{</span>
<span class="kt">int</span> <span class="n">count</span> <span class="o">=</span> <span class="n">Integer</span><span class="o">.</span><span class="na">parseInt</span><span class="o">(</span><span class="n">countStr</span><span class="o">);</span>
- <span class="k">this</span><span class="o">.</span><span class="na">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="nf">Values</span><span class="o">(</span><span class="n">wordName</span><span class="o">,</span> <span class="n">count</span><span class="o">));</span>
+ <span class="k">this</span><span class="o">.</span><span class="na">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">wordName</span><span class="o">,</span> <span class="n">count</span><span class="o">));</span>
<span class="c1">// print lookup result with low probability</span>
<span class="k">if</span><span class="o">(</span><span class="n">RANDOM</span><span class="o">.</span><span class="na">nextInt</span><span class="o">(</span><span class="mi">1000</span><span class="o">)</span> <span class="o">></span> <span class="mi">995</span><span class="o">)</span> <span class="o">{</span>
- <span class="n">LOG</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">"Lookup result - word : "</span> <span class="o">+</span> <span class="n">wordName</span> <span class="o">+</span> <span class="s">" / count : "</span> <span class="o">+</span> <span class="n">count</span><span class="o">);</span>
+ <span class="n">LOG</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">"Lookup result - word : "</span> <span class="o">+</span> <span class="n">wordName</span> <span class="o">+</span> <span class="s">" / count : "</span> <span class="o">+</span> <span class="n">count</span><span class="o">);</span>
<span class="o">}</span>
<span class="o">}</span> <span class="k">else</span> <span class="o">{</span>
<span class="c1">// skip</span>
- <span class="n">LOG</span><span class="o">.</span><span class="na">warn</span><span class="o">(</span><span class="s">"Word not found in Redis - word : "</span> <span class="o">+</span> <span class="n">wordName</span><span class="o">);</span>
+ <span class="n">LOG</span><span class="o">.</span><span class="na">warn</span><span class="o">(</span><span class="s">"Word not found in Redis - word : "</span> <span class="o">+</span> <span class="n">wordName</span><span class="o">);</span>
<span class="o">}</span>
<span class="o">}</span> <span class="k">finally</span> <span class="o">{</span>
<span class="k">if</span> <span class="o">(</span><span class="n">jedisCommands</span> <span class="o">!=</span> <span class="kc">null</span><span class="o">)</span> <span class="o">{</span>
@@ -232,11 +238,12 @@
<span class="o">}</span>
<span class="nd">@Override</span>
- <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
+ <span class="kd">public</span> <span class="kt">void</span> <span class="n">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span>
<span class="c1">// wordName, count</span>
- <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="nf">Fields</span><span class="o">(</span><span class="s">"wordName"</span><span class="o">,</span> <span class="s">"count"</span><span class="o">));</span>
+ <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"wordName"</span><span class="o">,</span> <span class="s">"count"</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
+
</code></pre></div>
<h3 id="trident-state-usage">Trident State usage</h3>
@@ -253,8 +260,8 @@
RedisStoreMapper storeMapper = new WordCountStoreMapper();
RedisLookupMapper lookupMapper = new WordCountLookupMapper();
RedisState.Factory factory = new RedisState.Factory(poolConfig);</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"> TridentTopology topology = new TridentTopology();
- Stream stream = topology.newStream("spout1", spout);
+<div class="highlight"><pre><code class="language-" data-lang=""> TridentTopology topology = new TridentTopology();
+ Stream stream = topology.newStream("spout1", spout);
stream.partitionPersist(factory,
fields,
@@ -262,14 +269,15 @@
new Fields());
TridentState state = topology.newStaticState(factory);
- stream = stream.stateQuery(state, new Fields("word"),
+ stream = stream.stateQuery(state, new Fields("word"),
new RedisStateQuerier(lookupMapper),
- new Fields("columnName","columnValue"));
-</code></pre></div><div class="highlight"><pre><code class="language-text" data-lang="text">RedisClusterState
+ new Fields("columnName","columnValue"));
+</code></pre></div><div class="highlight"><pre><code class="language-" data-lang="">
+RedisClusterState
```java
Set<InetSocketAddress> nodes = new HashSet<InetSocketAddress>();
- for (String hostPort : redisHostPort.split(",")) {
- String[] host_port = hostPort.split(":");
+ for (String hostPort : redisHostPort.split(",")) {
+ String[] host_port = hostPort.split(":");
nodes.add(new InetSocketAddress(host_port[0], Integer.valueOf(host_port[1])));
}
JedisClusterConfig clusterConfig = new JedisClusterConfig.Builder().setNodes(nodes)
@@ -279,7 +287,7 @@
RedisClusterState.Factory factory = new RedisClusterState.Factory(clusterConfig);
TridentTopology topology = new TridentTopology();
- Stream stream = topology.newStream("spout1", spout);
+ Stream stream = topology.newStream("spout1", spout);
stream.partitionPersist(factory,
fields,
@@ -287,9 +295,9 @@
new Fields());
TridentState state = topology.newStaticState(factory);
- stream = stream.stateQuery(state, new Fields("word"),
+ stream = stream.stateQuery(state, new Fields("word"),
new RedisClusterStateQuerier(lookupMapper),
- new Fields("columnName","columnValue"));
+ new Fields("columnName","columnValue"));
</code></pre></div>
<h2 id="license">License</h2>