You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/04/05 22:34:08 UTC
[12/14] storm git commit: STORM-2416 Release Packaging Improvements
STORM-2416 Release Packaging Improvements
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2a369e40
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2a369e40
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2a369e40
Branch: refs/heads/master
Commit: 2a369e408c18adf1045f57382d30fe18dacdbb19
Parents: 28c8711
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Mar 16 15:14:26 2017 -0400
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Apr 4 18:37:10 2017 +0900
----------------------------------------------------------------------
examples/flux-examples/README.md | 93 +
examples/flux-examples/pom.xml | 152 +
.../flux/examples/StatefulWordCounter.java | 64 +
.../storm/flux/examples/TestPrintBolt.java | 39 +
.../storm/flux/examples/TestWindowBolt.java | 47 +
.../storm/flux/examples/WordCountClient.java | 74 +
.../apache/storm/flux/examples/WordCounter.java | 71 +
.../src/main/resources/hbase_bolt.properties | 18 +
.../src/main/resources/hdfs_bolt.properties | 26 +
.../src/main/resources/kafka_spout.yaml | 136 +
.../src/main/resources/multilang.yaml | 89 +
.../src/main/resources/simple_hbase.yaml | 92 +
.../src/main/resources/simple_hdfs.yaml | 105 +
.../resources/simple_stateful_wordcount.yaml | 60 +
.../src/main/resources/simple_windowing.yaml | 69 +
.../src/main/resources/simple_wordcount.yaml | 68 +
examples/storm-elasticsearch-examples/pom.xml | 15 -
examples/storm-hbase-examples/pom.xml | 14 -
examples/storm-hdfs-examples/pom.xml | 14 -
examples/storm-hive-examples/pom.xml | 14 -
examples/storm-jdbc-examples/pom.xml | 14 -
examples/storm-jms-examples/README.markdown | 12 +
examples/storm-jms-examples/pom.xml | 150 +
.../storm/jms/example/ExampleJmsTopology.java | 131 +
.../apache/storm/jms/example/GenericBolt.java | 116 +
.../storm/jms/example/JsonTupleProducer.java | 58 +
.../storm/jms/example/SpringJmsProvider.java | 74 +
.../src/main/resources/jms-activemq.xml | 53 +
.../src/main/resources/log4j.properties | 29 +
examples/storm-kafka-client-examples/pom.xml | 14 -
examples/storm-kafka-examples/pom.xml | 14 -
examples/storm-mongodb-examples/pom.xml | 14 -
examples/storm-mqtt-examples/pom.xml | 14 -
examples/storm-opentsdb-examples/pom.xml | 14 -
examples/storm-perf/README.markdown | 50 +
examples/storm-perf/pom.xml | 107 +
.../main/conf/ConstSpoutIdBoltNullBoltTopo.yaml | 22 +
.../src/main/conf/ConstSpoutNullBoltTopo.yaml | 22 +
.../src/main/conf/FileReadWordCountTopo.yaml | 23 +
.../src/main/conf/HdfsSpoutNullBoltTopo.yaml | 25 +
.../storm-perf/src/main/conf/KafkaHdfsTopo.yaml | 26 +
.../src/main/conf/KafkaSpoutNullBoltTopo.yaml | 23 +
.../src/main/conf/StrGenSpoutHdfsBoltTopo.yaml | 25 +
.../perf/ConstSpoutIdBoltNullBoltTopo.java | 101 +
.../storm/perf/ConstSpoutNullBoltTopo.java | 107 +
.../apache/storm/perf/ConstSpoutOnlyTopo.java | 74 +
.../storm/perf/FileReadWordCountTopo.java | 96 +
.../storm/perf/HdfsSpoutNullBoltTopo.java | 101 +
.../org/apache/storm/perf/KafkaHdfsTopo.java | 168 +
.../storm/perf/KafkaSpoutNullBoltTopo.java | 114 +
.../storm/perf/StrGenSpoutHdfsBoltTopo.java | 154 +
.../org/apache/storm/perf/bolt/CountBolt.java | 58 +
.../org/apache/storm/perf/bolt/DevNullBolt.java | 47 +
.../java/org/apache/storm/perf/bolt/IdBolt.java | 49 +
.../storm/perf/bolt/SplitSentenceBolt.java | 58 +
.../org/apache/storm/perf/spout/ConstSpout.java | 70 +
.../apache/storm/perf/spout/FileReadSpout.java | 141 +
.../apache/storm/perf/spout/StringGenSpout.java | 93 +
.../storm/perf/utils/BasicMetricsCollector.java | 309 +
.../org/apache/storm/perf/utils/Helper.java | 131 +
.../apache/storm/perf/utils/IdentityBolt.java | 51 +
.../apache/storm/perf/utils/MetricsSample.java | 248 +
.../src/main/sampledata/randomwords.txt | 14049 +++++++++++++++++
examples/storm-pmml-examples/pom.xml | 14 -
examples/storm-redis-examples/pom.xml | 14 -
examples/storm-solr-examples/pom.xml | 14 -
examples/storm-starter/pom.xml | 11 -
external/flux/flux-examples/README.md | 93 -
external/flux/flux-examples/pom.xml | 146 -
.../flux/examples/StatefulWordCounter.java | 64 -
.../storm/flux/examples/TestPrintBolt.java | 39 -
.../storm/flux/examples/TestWindowBolt.java | 47 -
.../storm/flux/examples/WordCountClient.java | 74 -
.../apache/storm/flux/examples/WordCounter.java | 71 -
.../src/main/resources/hbase_bolt.properties | 18 -
.../src/main/resources/hdfs_bolt.properties | 26 -
.../src/main/resources/kafka_spout.yaml | 136 -
.../src/main/resources/multilang.yaml | 89 -
.../src/main/resources/simple_hbase.yaml | 92 -
.../src/main/resources/simple_hdfs.yaml | 105 -
.../resources/simple_stateful_wordcount.yaml | 60 -
.../src/main/resources/simple_windowing.yaml | 69 -
.../src/main/resources/simple_wordcount.yaml | 68 -
external/flux/pom.xml | 21 -
external/sql/storm-sql-core/pom.xml | 4 +-
.../storm-sql-external/storm-sql-hdfs/pom.xml | 6 +-
.../storm-sql-external/storm-sql-kafka/pom.xml | 6 +-
.../storm-sql-mongodb/pom.xml | 6 +-
.../storm-sql-external/storm-sql-redis/pom.xml | 6 +-
external/sql/storm-sql-runtime/pom.xml | 2 +-
external/storm-cassandra/pom.xml | 2 +-
external/storm-druid/pom.xml | 42 +-
external/storm-elasticsearch/pom.xml | 2 +-
external/storm-eventhubs/pom.xml | 86 +-
external/storm-hbase/pom.xml | 2 +-
external/storm-hdfs/pom.xml | 2 +-
external/storm-hive/pom.xml | 2 +-
external/storm-jdbc/pom.xml | 2 +-
external/storm-jms/core/pom.xml | 95 -
.../apache/storm/jms/JmsMessageProducer.java | 46 -
.../java/org/apache/storm/jms/JmsProvider.java | 48 -
.../org/apache/storm/jms/JmsTupleProducer.java | 58 -
.../java/org/apache/storm/jms/bolt/JmsBolt.java | 219 -
.../apache/storm/jms/spout/JmsMessageID.java | 58 -
.../org/apache/storm/jms/spout/JmsSpout.java | 382 -
.../org/apache/storm/jms/trident/JmsBatch.java | 27 -
.../org/apache/storm/jms/trident/JmsState.java | 129 -
.../storm/jms/trident/JmsStateFactory.java | 40 -
.../apache/storm/jms/trident/JmsUpdater.java | 38 -
.../storm/jms/trident/TridentJmsSpout.java | 409 -
.../apache/storm/jms/spout/JmsSpoutTest.java | 88 -
.../apache/storm/jms/spout/MockJmsProvider.java | 62 -
.../jms/spout/MockSpoutOutputCollector.java | 55 -
.../storm/jms/spout/MockTupleProducer.java | 47 -
.../core/src/test/resources/jndi.properties | 18 -
external/storm-jms/examples/README.markdown | 12 -
external/storm-jms/examples/pom.xml | 150 -
.../storm/jms/example/ExampleJmsTopology.java | 131 -
.../apache/storm/jms/example/GenericBolt.java | 116 -
.../storm/jms/example/JsonTupleProducer.java | 58 -
.../storm/jms/example/SpringJmsProvider.java | 74 -
.../src/main/resources/jms-activemq.xml | 53 -
.../src/main/resources/log4j.properties | 29 -
external/storm-jms/pom.xml | 48 +-
.../apache/storm/jms/JmsMessageProducer.java | 46 +
.../java/org/apache/storm/jms/JmsProvider.java | 48 +
.../org/apache/storm/jms/JmsTupleProducer.java | 58 +
.../java/org/apache/storm/jms/bolt/JmsBolt.java | 219 +
.../apache/storm/jms/spout/JmsMessageID.java | 58 +
.../org/apache/storm/jms/spout/JmsSpout.java | 382 +
.../org/apache/storm/jms/trident/JmsBatch.java | 27 +
.../org/apache/storm/jms/trident/JmsState.java | 129 +
.../storm/jms/trident/JmsStateFactory.java | 40 +
.../apache/storm/jms/trident/JmsUpdater.java | 38 +
.../storm/jms/trident/TridentJmsSpout.java | 409 +
.../apache/storm/jms/spout/JmsSpoutTest.java | 88 +
.../apache/storm/jms/spout/MockJmsProvider.java | 62 +
.../jms/spout/MockSpoutOutputCollector.java | 55 +
.../storm/jms/spout/MockTupleProducer.java | 47 +
.../src/test/resources/jndi.properties | 18 +
external/storm-kafka-client/pom.xml | 2 +-
external/storm-kafka/pom.xml | 2 +-
external/storm-kinesis/pom.xml | 2 +-
external/storm-metrics/pom.xml | 2 +-
external/storm-mongodb/pom.xml | 148 +-
external/storm-mqtt/core/pom.xml | 126 -
.../java/org/apache/storm/mqtt/MqttLogger.java | 36 -
.../java/org/apache/storm/mqtt/MqttMessage.java | 41 -
.../apache/storm/mqtt/MqttMessageMapper.java | 44 -
.../org/apache/storm/mqtt/MqttTupleMapper.java | 37 -
.../org/apache/storm/mqtt/bolt/MqttBolt.java | 105 -
.../apache/storm/mqtt/common/MqttOptions.java | 334 -
.../apache/storm/mqtt/common/MqttPublisher.java | 67 -
.../org/apache/storm/mqtt/common/MqttUtils.java | 88 -
.../org/apache/storm/mqtt/common/SslUtils.java | 64 -
.../mqtt/mappers/ByteArrayMessageMapper.java | 34 -
.../storm/mqtt/mappers/StringMessageMapper.java | 37 -
.../apache/storm/mqtt/spout/AckableMessage.java | 71 -
.../org/apache/storm/mqtt/spout/MqttSpout.java | 262 -
.../storm/mqtt/ssl/DefaultKeyStoreLoader.java | 97 -
.../apache/storm/mqtt/ssl/KeyStoreLoader.java | 35 -
.../storm/mqtt/trident/MqttPublishFunction.java | 85 -
.../storm/mqtt/StormMqttIntegrationTest.java | 153 -
external/storm-mqtt/pom.xml | 144 +-
.../java/org/apache/storm/mqtt/MqttLogger.java | 36 +
.../java/org/apache/storm/mqtt/MqttMessage.java | 41 +
.../apache/storm/mqtt/MqttMessageMapper.java | 44 +
.../org/apache/storm/mqtt/MqttTupleMapper.java | 37 +
.../org/apache/storm/mqtt/bolt/MqttBolt.java | 105 +
.../apache/storm/mqtt/common/MqttOptions.java | 334 +
.../apache/storm/mqtt/common/MqttPublisher.java | 67 +
.../org/apache/storm/mqtt/common/MqttUtils.java | 88 +
.../org/apache/storm/mqtt/common/SslUtils.java | 64 +
.../mqtt/mappers/ByteArrayMessageMapper.java | 34 +
.../storm/mqtt/mappers/StringMessageMapper.java | 37 +
.../apache/storm/mqtt/spout/AckableMessage.java | 71 +
.../org/apache/storm/mqtt/spout/MqttSpout.java | 262 +
.../storm/mqtt/ssl/DefaultKeyStoreLoader.java | 97 +
.../apache/storm/mqtt/ssl/KeyStoreLoader.java | 35 +
.../storm/mqtt/trident/MqttPublishFunction.java | 85 +
.../storm/mqtt/StormMqttIntegrationTest.java | 153 +
external/storm-opentsdb/pom.xml | 2 +-
external/storm-pmml/pom.xml | 2 +-
external/storm-redis/pom.xml | 2 +-
external/storm-solr/pom.xml | 2 +-
pom.xml | 18 +-
.../final-package/src/main/assembly/binary.xml | 254 +-
storm-perf/README.markdown | 50 -
storm-perf/pom.xml | 122 -
.../main/conf/ConstSpoutIdBoltNullBoltTopo.yaml | 22 -
.../src/main/conf/ConstSpoutNullBoltTopo.yaml | 22 -
.../src/main/conf/FileReadWordCountTopo.yaml | 23 -
.../src/main/conf/HdfsSpoutNullBoltTopo.yaml | 25 -
storm-perf/src/main/conf/KafkaHdfsTopo.yaml | 26 -
.../src/main/conf/KafkaSpoutNullBoltTopo.yaml | 23 -
.../src/main/conf/StrGenSpoutHdfsBoltTopo.yaml | 25 -
.../perf/ConstSpoutIdBoltNullBoltTopo.java | 101 -
.../storm/perf/ConstSpoutNullBoltTopo.java | 107 -
.../apache/storm/perf/ConstSpoutOnlyTopo.java | 74 -
.../storm/perf/FileReadWordCountTopo.java | 96 -
.../storm/perf/HdfsSpoutNullBoltTopo.java | 101 -
.../org/apache/storm/perf/KafkaHdfsTopo.java | 168 -
.../storm/perf/KafkaSpoutNullBoltTopo.java | 114 -
.../storm/perf/StrGenSpoutHdfsBoltTopo.java | 154 -
.../org/apache/storm/perf/bolt/CountBolt.java | 58 -
.../org/apache/storm/perf/bolt/DevNullBolt.java | 47 -
.../java/org/apache/storm/perf/bolt/IdBolt.java | 49 -
.../storm/perf/bolt/SplitSentenceBolt.java | 58 -
.../org/apache/storm/perf/spout/ConstSpout.java | 70 -
.../apache/storm/perf/spout/FileReadSpout.java | 141 -
.../apache/storm/perf/spout/StringGenSpout.java | 93 -
.../storm/perf/utils/BasicMetricsCollector.java | 309 -
.../org/apache/storm/perf/utils/Helper.java | 131 -
.../apache/storm/perf/utils/IdentityBolt.java | 51 -
.../apache/storm/perf/utils/MetricsSample.java | 248 -
storm-perf/src/main/sampledata/randomwords.txt | 14049 -----------------
216 files changed, 21999 insertions(+), 22608 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/README.md
----------------------------------------------------------------------
diff --git a/examples/flux-examples/README.md b/examples/flux-examples/README.md
new file mode 100644
index 0000000..3d610b4
--- /dev/null
+++ b/examples/flux-examples/README.md
@@ -0,0 +1,93 @@
+# Flux Examples
+A collection of examples illustrating various capabilities.
+
+## Building From Source and Running
+
+Checkout the projects source and perform a top level Maven build (i.e. from the `flux` directory):
+
+```bash
+git clone https://github.com/apache/storm.git
+cd storm
+mvn install -DskipTests=true
+```
+
+This will create a shaded (i.e. "fat" or "uber") jar in the `external/flux/flux-examples/target` directory that can run/deployed with
+the `storm` command:
+
+```bash
+cd flux-examples
+storm jar ./target/flux-examples-*-SNAPSHOT.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_wordcount.yaml
+```
+
+The example YAML files are also packaged in the examples jar, so they can also be referenced with Flux's `--resource`
+command line switch:
+
+```bash
+storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local --resource /simple_wordcount.yaml
+```
+
+## Available Examples
+
+### [simple_wordcount.yaml](src/main/resources/simple_wordcount.yaml)
+
+This is a very basic wordcount example using Java spouts and bolts. It simply logs the running count of each word
+received.
+
+### [multilang.yaml](src/main/resources/multilang.yaml)
+
+Another wordcount example that uses a spout written in JavaScript (node.js), a bolt written in Python, and two bolts
+written in java.
+
+### [kafka_spout.yaml](src/main/resources/kafka_spout.yaml)
+
+This example illustrates how to configure Storm's `storm-kafka` spout using Flux YAML DSL `components`, `references`,
+and `constructor arguments` constructs.
+
+### [simple_hdfs.yaml](src/main/resources/simple_hdfs.yaml)
+
+This example demonstrates using Flux to setup a storm-hdfs bolt to write to an HDFS cluster. It also demonstrates Flux's
+variable substitution/filtering feature.
+
+To run the `simple_hdfs.yaml` example, copy the `hdfs_bolt.properties` file to a convenient location and change, at
+least, the property `hdfs.url` to point to a HDFS cluster. Then you can run the example something like:
+
+```bash
+storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hdfs.yaml --filter my_hdfs_bolt.properties
+```
+
+### [simple_hbase.yaml](src/main/resources/simple_hbase.yaml)
+
+This example illustrates how to use Flux to setup a storm-hbase bolt to write to HBase.
+
+To run the `simple_hbase.yaml` example, copy the `hbase_bolt.properties` file to a convenient location and change the properties
+ `hbase.rootdir` and `hbase.zookeeper.quorum`. Then you can run the example something like:
+
+```bash
+storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_hbase.yaml --filter my_hbase_bolt.properties
+```
+
+### [simple_windowing.yaml](src/main/resources/simple_windowing.yaml)
+
+This example illustrates how to use Flux to set up a storm topology that contains windowing operations.
+
+To run,
+
+```bash
+storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_windowing.yaml
+```
+
+### [simple_stateful_wordcount.yaml](src/main/resources/simple_stateful_wordcount.yaml)
+
+Flux also supports stateful bolts which is illustrated with this example. It is basically an extension of the basic wordcount example.
+The state is periodically saved (checkpointed) and restored when the topology is restarted.
+
+```bash
+storm jar ./target/flux-examples-*.jar org.apache.storm.flux.Flux --local ./src/main/resources/simple_stateful_wordcount.yaml
+```
+
+By default the state is stored in-memory only. As such you won't see a resumed state unless you configure to use Redis as the state backend.
+Ensure that you have Redis running at `localhost:6379` and that `storm-redis-*.jar` is in the classpath.
+
+```bash
+STORM_EXT_CLASSPATH=../../storm-redis/target storm jar ./target/flux-examples-*.jar -c topology.state.provider=org.apache.storm.redis.state.RedisKeyValueStateProvider org.apache.storm.flux.Flux --local ./src/main/resources/simple_stateful_wordcount.yaml
+```
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/flux-examples/pom.xml b/examples/flux-examples/pom.xml
new file mode 100644
index 0000000..14ad00c
--- /dev/null
+++ b/examples/flux-examples/pom.xml
@@ -0,0 +1,152 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>storm</artifactId>
+ <groupId>org.apache.storm</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>flux-examples</artifactId>
+ <packaging>jar</packaging>
+
+ <name>flux-examples</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${project.version}</version>
+ <scope>${provided.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>flux-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>flux-wrappers</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-hdfs</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-hbase</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-kafka</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>${storm.kafka.artifact.id}</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.sf</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.dsa</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/*.rsa</exclude>
+ <exclude>META-INF/*.EC</exclude>
+ <exclude>META-INF/*.ec</exclude>
+ <exclude>META-INF/MSFTSIG.SF</exclude>
+ <exclude>META-INF/MSFTSIG.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.apache.storm.flux.Flux</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java
new file mode 100644
index 0000000..5534888
--- /dev/null
+++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.flux.examples;
+
+import org.apache.storm.state.KeyValueState;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.base.BaseStatefulBolt;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
+
+import java.util.Map;
+
+public class StatefulWordCounter extends BaseStatefulBolt<KeyValueState<String, Long>> {
+
+ private KeyValueState<String, Long> wordCounts;
+ private OutputCollector collector;
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ this.collector = collector;
+ }
+
+ @Override
+ public void initState(KeyValueState<String, Long> state) {
+ wordCounts = state;
+ }
+
+ @Override
+ public void execute(Tuple tuple) {
+ String word = tuple.getString(0);
+
+ Long count = wordCounts.get(word, 0L);
+ count++;
+ wordCounts.put(word, count);
+
+ collector.emit(tuple, new Values(word, count));
+ collector.ack(tuple);
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word", "count"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java
new file mode 100644
index 0000000..137e354
--- /dev/null
+++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.flux.examples;
+
+import org.apache.storm.topology.BasicOutputCollector;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseBasicBolt;
+import org.apache.storm.tuple.Tuple;
+
+/**
+ * Prints the tuples to stdout
+ */
+public class TestPrintBolt extends BaseBasicBolt {
+
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ System.out.println(tuple);
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer ofd) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java
new file mode 100644
index 0000000..8c904d9
--- /dev/null
+++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.flux.examples;
+
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseWindowedBolt;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
+import org.apache.storm.windowing.TupleWindow;
+
+import java.util.Map;
+
+public class TestWindowBolt extends BaseWindowedBolt {
+ private OutputCollector collector;
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ this.collector = collector;
+ }
+
+ @Override
+ public void execute(TupleWindow inputWindow) {
+ collector.emit(new Values(inputWindow.get().size()));
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("count"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java
new file mode 100644
index 0000000..eb4fb7a
--- /dev/null
+++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.flux.examples;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.io.FileInputStream;
+import java.util.Properties;
+
+/**
+ * Connects to the 'WordCount' HBase table and prints counts for each word.
+ *
+ * Assumes you have run (or are running) the YAML topology definition in
+ * <code>simple_hbase.yaml</code>
+ *
+ * You will also need to modify `src/main/resources/hbase-site.xml`
+ * to point to your HBase instance, and then repackage with `mvn package`.
+ * This is a known issue.
+ *
+ */
+public class WordCountClient {
+
+ public static void main(String[] args) throws Exception {
+ Configuration config = HBaseConfiguration.create();
+ if(args.length == 1){
+ Properties props = new Properties();
+ props.load(new FileInputStream(args[0]));
+ System.out.println("HBase configuration:");
+ for(Object key : props.keySet()) {
+ System.out.println(key + "=" + props.get(key));
+ config.set((String)key, props.getProperty((String)key));
+ }
+ } else {
+ System.out.println("Usage: WordCountClient <hbase_config.properties>");
+ System.exit(1);
+ }
+
+ HTable table = new HTable(config, "WordCount");
+ String[] words = new String[] {"nathan", "mike", "jackson", "golda", "bertels"};
+
+ for (String word : words) {
+ Get get = new Get(Bytes.toBytes(word));
+ Result result = table.get(get);
+
+ byte[] countBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("count"));
+ byte[] wordBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("word"));
+
+ String wordStr = Bytes.toString(wordBytes);
+ long count = Bytes.toLong(countBytes);
+ System.out.println("Word: '" + wordStr + "', Count: " + count);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java
new file mode 100644
index 0000000..7093105
--- /dev/null
+++ b/examples/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.flux.examples;
+
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.BasicOutputCollector;
+import org.apache.storm.topology.IBasicBolt;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseBasicBolt;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+import static org.apache.storm.utils.Utils.tuple;
+
+/**
+ * This bolt is used by the HBase example. It simply emits the first field
+ * found in the incoming tuple as "word", with a "count" of `1`.
+ *
+ * In this case, the downstream HBase bolt handles the counting, so a value
+ * of `1` will just increment the HBase counter by one.
+ */
+public class WordCounter extends BaseBasicBolt {
+ private static final Logger LOG = LoggerFactory.getLogger(WordCounter.class);
+
+
+
+ @SuppressWarnings("rawtypes")
+ public void prepare(Map stormConf, TopologyContext context) {
+ }
+
+ /*
+ * Just output the word value with a count of 1.
+ * The HBaseBolt will handle incrementing the counter.
+ */
+ public void execute(Tuple input, BasicOutputCollector collector) {
+ collector.emit(tuple(input.getValues().get(0), 1));
+ }
+
+ public void cleanup() {
+
+ }
+
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word", "count"));
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/resources/hbase_bolt.properties
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/resources/hbase_bolt.properties b/examples/flux-examples/src/main/resources/hbase_bolt.properties
new file mode 100644
index 0000000..f8ed50c
--- /dev/null
+++ b/examples/flux-examples/src/main/resources/hbase_bolt.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hbase.rootdir=hdfs://hadoop:54310/hbase
+hbase.zookeeper.quorum=hadoop
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/resources/hdfs_bolt.properties
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/resources/hdfs_bolt.properties b/examples/flux-examples/src/main/resources/hdfs_bolt.properties
new file mode 100644
index 0000000..7bcbe7a
--- /dev/null
+++ b/examples/flux-examples/src/main/resources/hdfs_bolt.properties
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# The HDFS url
+hdfs.url=hdfs://hadoop:54310
+
+# The HDFS directory where the bolt will write incoming data
+hdfs.write.dir=/incoming
+
+# The HDFS directory where files will be moved once the bolt has
+# finished writing to it.
+hdfs.dest.dir=/complete
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/resources/kafka_spout.yaml
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/resources/kafka_spout.yaml b/examples/flux-examples/src/main/resources/kafka_spout.yaml
new file mode 100644
index 0000000..7533ce4
--- /dev/null
+++ b/examples/flux-examples/src/main/resources/kafka_spout.yaml
@@ -0,0 +1,136 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Test ability to wire together shell spouts/bolts
+---
+
+# topology definition
+# name to be used when submitting
+name: "kafka-topology"
+
+# Components
+# Components are analagous to Spring beans. They are meant to be used as constructor,
+# property(setter), and builder arguments.
+#
+# for the time being, components must be declared in the order they are referenced
+components:
+ - id: "stringScheme"
+ className: "org.apache.storm.kafka.StringScheme"
+
+ - id: "stringMultiScheme"
+ className: "org.apache.storm.spout.SchemeAsMultiScheme"
+ constructorArgs:
+ - ref: "stringScheme"
+
+ - id: "zkHosts"
+ className: "org.apache.storm.kafka.ZkHosts"
+ constructorArgs:
+ - "localhost:2181"
+
+# Alternative kafka config
+# - id: "kafkaConfig"
+# className: "org.apache.storm.kafka.KafkaConfig"
+# constructorArgs:
+# # brokerHosts
+# - ref: "zkHosts"
+# # topic
+# - "myKafkaTopic"
+# # clientId (optional)
+# - "myKafkaClientId"
+
+ - id: "spoutConfig"
+ className: "org.apache.storm.kafka.SpoutConfig"
+ constructorArgs:
+ # brokerHosts
+ - ref: "zkHosts"
+ # topic
+ - "myKafkaTopic"
+ # zkRoot
+ - "/kafkaSpout"
+ # id
+ - "myId"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "scheme"
+ ref: "stringMultiScheme"
+
+
+
+# NOTE: We may want to consider some level of spring integration. For example, allowing component references
+# to a spring `ApplicationContext`.
+
+# topology configuration
+# this will be passed to the submitter as a map of config options
+#
+config:
+ topology.workers: 1
+ # ...
+
+# spout definitions
+spouts:
+ - id: "kafka-spout"
+ className: "org.apache.storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "spoutConfig"
+
+# bolt definitions
+bolts:
+ - id: "splitsentence"
+ className: "org.apache.storm.flux.wrappers.bolts.FluxShellBolt"
+ constructorArgs:
+ # command line
+ - ["python", "splitsentence.py"]
+ # output fields
+ - ["word"]
+ parallelism: 1
+ # ...
+
+ - id: "log"
+ className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt"
+ parallelism: 1
+ # ...
+
+ - id: "count"
+ className: "org.apache.storm.testing.TestWordCounter"
+ parallelism: 1
+ # ...
+
+#stream definitions
+# stream definitions define connections between spouts and bolts.
+# note that such connections can be cyclical
+# custom stream groupings are also supported
+
+streams:
+ - name: "kafka --> split" # name isn't used (placeholder for logging, UI, etc.)
+ from: "kafka-spout"
+ to: "splitsentence"
+ grouping:
+ type: SHUFFLE
+
+ - name: "split --> count"
+ from: "splitsentence"
+ to: "count"
+ grouping:
+ type: FIELDS
+ args: ["word"]
+
+ - name: "count --> log"
+ from: "count"
+ to: "log"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/resources/multilang.yaml
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/resources/multilang.yaml b/examples/flux-examples/src/main/resources/multilang.yaml
new file mode 100644
index 0000000..aaab5d3
--- /dev/null
+++ b/examples/flux-examples/src/main/resources/multilang.yaml
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test ability to wire together shell spouts/bolts
+---
+
+# topology definition
+# name to be used when submitting
+name: "shell-topology"
+
+# topology configuration
+# this will be passed to the submitter as a map of config options
+#
+config:
+ topology.workers: 1
+ # ...
+
+# spout definitions
+spouts:
+ - id: "sentence-spout"
+ className: "org.apache.storm.flux.wrappers.spouts.FluxShellSpout"
+ # shell spout constructor takes 2 arguments: String[], String[]
+ constructorArgs:
+ # command line
+ - ["node", "randomsentence.js"]
+ # output fields
+ - ["word"]
+ parallelism: 1
+ # ...
+
+# bolt definitions
+bolts:
+ - id: "splitsentence"
+ className: "org.apache.storm.flux.wrappers.bolts.FluxShellBolt"
+ constructorArgs:
+ # command line
+ - ["python", "splitsentence.py"]
+ # output fields
+ - ["word"]
+ parallelism: 1
+ # ...
+
+ - id: "log"
+ className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt"
+ parallelism: 1
+ # ...
+
+ - id: "count"
+ className: "org.apache.storm.testing.TestWordCounter"
+ parallelism: 1
+ # ...
+
+#stream definitions
+# stream definitions define connections between spouts and bolts.
+# note that such connections can be cyclical
+# custom stream groupings are also supported
+
+streams:
+ - name: "spout --> split" # name isn't used (placeholder for logging, UI, etc.)
+ from: "sentence-spout"
+ to: "splitsentence"
+ grouping:
+ type: SHUFFLE
+
+ - name: "split --> count"
+ from: "splitsentence"
+ to: "count"
+ grouping:
+ type: FIELDS
+ args: ["word"]
+
+ - name: "count --> log"
+ from: "count"
+ to: "log"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/resources/simple_hbase.yaml
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/resources/simple_hbase.yaml b/examples/flux-examples/src/main/resources/simple_hbase.yaml
new file mode 100644
index 0000000..93a2781
--- /dev/null
+++ b/examples/flux-examples/src/main/resources/simple_hbase.yaml
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+# NOTE: To use this example, you will need to modify `src/main/resources/hbase-site.xml`
+# to point to your HBase instance, and then repackage with `mvn package`.
+# This is a known issue.
+
+# topology definition
+# name to be used when submitting
+name: "hbase-persistent-wordcount"
+
+# Components
+components:
+ - id: "columnFields"
+ className: "org.apache.storm.tuple.Fields"
+ constructorArgs:
+ - ["word"]
+
+ - id: "counterFields"
+ className: "org.apache.storm.tuple.Fields"
+ constructorArgs:
+ - ["count"]
+
+ - id: "mapper"
+ className: "org.apache.storm.hbase.bolt.mapper.SimpleHBaseMapper"
+ configMethods:
+ - name: "withRowKeyField"
+ args: ["word"]
+ - name: "withColumnFields"
+ args: [ref: "columnFields"]
+ - name: "withCounterFields"
+ args: [ref: "counterFields"]
+ - name: "withColumnFamily"
+ args: ["cf"]
+
+# topology configuration
+# this will be passed to the submitter as a map of config options
+config:
+ topology.workers: 1
+ hbase.conf:
+ hbase.rootdir: "${hbase.rootdir}"
+ hbase.zookeeper.quorum: "${hbase.zookeeper.quorum}"
+
+# spout definitions
+spouts:
+ - id: "word-spout"
+ className: "org.apache.storm.testing.TestWordSpout"
+ parallelism: 1
+
+# bolt definitions
+
+bolts:
+ - id: "count-bolt"
+ className: "org.apache.storm.flux.examples.WordCounter"
+ parallelism: 1
+
+ - id: "hbase-bolt"
+ className: "org.apache.storm.hbase.bolt.HBaseBolt"
+ constructorArgs:
+ - "WordCount" # HBase table name
+ - ref: "mapper"
+ configMethods:
+ - name: "withConfigKey"
+ args: ["hbase.conf"]
+ parallelism: 1
+
+streams:
+ - name: "" # name isn't used (placeholder for logging, UI, etc.)
+ from: "word-spout"
+ to: "count-bolt"
+ grouping:
+ type: SHUFFLE
+
+ - name: "" # name isn't used (placeholder for logging, UI, etc.)
+ from: "count-bolt"
+ to: "hbase-bolt"
+ grouping:
+ type: FIELDS
+ args: ["word"]
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/resources/simple_hdfs.yaml
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/resources/simple_hdfs.yaml b/examples/flux-examples/src/main/resources/simple_hdfs.yaml
new file mode 100644
index 0000000..b8d4020
--- /dev/null
+++ b/examples/flux-examples/src/main/resources/simple_hdfs.yaml
@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test ability to wire together shell spouts/bolts
+---
+
+# topology definition
+# name to be used when submitting
+name: "hdfs-topology"
+
+# Components
+# Components are analagous to Spring beans. They are meant to be used as constructor,
+# property(setter), and builder arguments.
+#
+# for the time being, components must be declared in the order they are referenced
+components:
+ - id: "syncPolicy"
+ className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy"
+ constructorArgs:
+ - 1000
+ - id: "rotationPolicy"
+ className: "org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy"
+ constructorArgs:
+ - 30
+ - SECONDS
+
+ - id: "fileNameFormat"
+ className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
+ configMethods:
+ - name: "withPath"
+ args: ["${hdfs.write.dir}"]
+ - name: "withExtension"
+ args: [".txt"]
+
+ - id: "recordFormat"
+ className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat"
+ configMethods:
+ - name: "withFieldDelimiter"
+ args: ["|"]
+
+ - id: "rotationAction"
+ className: "org.apache.storm.hdfs.common.rotation.MoveFileAction"
+ configMethods:
+ - name: "toDestination"
+ args: ["${hdfs.dest.dir}"]
+
+# spout definitions
+spouts:
+ - id: "spout-1"
+ className: "org.apache.storm.testing.TestWordSpout"
+ parallelism: 1
+ # ...
+
+# bolt definitions
+
+bolts:
+ - id: "bolt-1"
+ className: "org.apache.storm.hdfs.bolt.HdfsBolt"
+ configMethods:
+ - name: "withConfigKey"
+ args: ["hdfs.config"]
+ - name: "withFsUrl"
+ args: ["${hdfs.url}"]
+ - name: "withFileNameFormat"
+ args: [ref: "fileNameFormat"]
+ - name: "withRecordFormat"
+ args: [ref: "recordFormat"]
+ - name: "withRotationPolicy"
+ args: [ref: "rotationPolicy"]
+ - name: "withSyncPolicy"
+ args: [ref: "syncPolicy"]
+ - name: "addRotationAction"
+ args: [ref: "rotationAction"]
+ parallelism: 1
+ # ...
+
+ - id: "bolt-2"
+ className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt"
+ parallelism: 1
+
+streams:
+ - name: "" # name isn't used (placeholder for logging, UI, etc.)
+ from: "spout-1"
+ to: "bolt-1"
+ grouping:
+ type: SHUFFLE
+
+ - name: "" # name isn't used (placeholder for logging, UI, etc.)
+ from: "spout-1"
+ to: "bolt-2"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/resources/simple_stateful_wordcount.yaml
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/resources/simple_stateful_wordcount.yaml b/examples/flux-examples/src/main/resources/simple_stateful_wordcount.yaml
new file mode 100644
index 0000000..14b9b3a
--- /dev/null
+++ b/examples/flux-examples/src/main/resources/simple_stateful_wordcount.yaml
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+# topology definition
+# name to be used when submitting
+name: "stateful-wordcount-topology"
+
+# topology configuration
+# this will be passed to the submitter as a map of config options
+#
+config:
+ topology.workers: 1
+
+# spout definitions
+spouts:
+ - id: "spout-1"
+ className: "org.apache.storm.testing.TestWordSpout"
+ parallelism: 1
+
+# bolt definitions
+bolts:
+ - id: "bolt-1"
+ className: "org.apache.storm.flux.examples.StatefulWordCounter"
+ parallelism: 1
+
+ - id: "bolt-2"
+ className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt"
+ parallelism: 1
+
+#stream definitions
+# stream definitions define connections between spouts and bolts.
+# note that such connections can be cyclical
+streams:
+ - name: "spout-1 --> bolt-1" # name isn't used (placeholder for logging, UI, etc.)
+ from: "spout-1"
+ to: "bolt-1"
+ grouping:
+ type: FIELDS
+ args: ["word"]
+
+ - name: "bolt-1 --> bolt2"
+ from: "bolt-1"
+ to: "bolt-2"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/resources/simple_windowing.yaml
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/resources/simple_windowing.yaml b/examples/flux-examples/src/main/resources/simple_windowing.yaml
new file mode 100755
index 0000000..f2f74ff
--- /dev/null
+++ b/examples/flux-examples/src/main/resources/simple_windowing.yaml
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+name: "sliding-window-topology"
+
+components:
+ - id: "windowLength"
+ className: "org.apache.storm.topology.base.BaseWindowedBolt$Count"
+ constructorArgs:
+ - 5
+ - id: "slidingInterval"
+ className: "org.apache.storm.topology.base.BaseWindowedBolt$Count"
+ constructorArgs:
+ - 3
+
+config:
+ topology.workers: 1
+
+# spout definitions
+spouts:
+ - id: "spout-1"
+ className: "org.apache.storm.testing.TestWordSpout"
+ parallelism: 1
+
+# bolt definitions
+bolts:
+ - id: "bolt-1"
+ className: "org.apache.storm.flux.examples.TestWindowBolt"
+ configMethods:
+ - name: "withWindow"
+ args: [ref: "windowLength", ref: "slidingInterval"]
+ parallelism: 1
+ - id: "bolt-2"
+ className: "org.apache.storm.flux.examples.TestPrintBolt"
+ parallelism: 1
+
+
+#stream definitions
+# stream definitions define connections between spouts and bolts.
+# note that such connections can be cyclical
+streams:
+ - name: "spout-1 --> bolt-1" # name isn't used (placeholder for logging, UI, etc.)
+# id: "connection-1"
+ from: "spout-1"
+ to: "bolt-1"
+ grouping:
+ type: FIELDS
+ args: ["word"]
+ - name: "bolt-1 --> bolt-2" # name isn't used (placeholder for logging, UI, etc.)
+# id: "connection-1"
+ from: "bolt-1"
+ to: "bolt-2"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/flux-examples/src/main/resources/simple_wordcount.yaml
----------------------------------------------------------------------
diff --git a/examples/flux-examples/src/main/resources/simple_wordcount.yaml b/examples/flux-examples/src/main/resources/simple_wordcount.yaml
new file mode 100644
index 0000000..6443a97
--- /dev/null
+++ b/examples/flux-examples/src/main/resources/simple_wordcount.yaml
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+# topology definition
+# name to be used when submitting
+name: "yaml-topology"
+
+# topology configuration
+# this will be passed to the submitter as a map of config options
+#
+config:
+ topology.workers: 1
+
+# spout definitions
+spouts:
+ - id: "spout-1"
+ className: "org.apache.storm.testing.TestWordSpout"
+ parallelism: 1
+
+# bolt definitions
+bolts:
+ - id: "bolt-1"
+ className: "org.apache.storm.testing.TestWordCounter"
+ parallelism: 1
+
+ - id: "bolt-2"
+ className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt"
+ parallelism: 1
+
+#stream definitions
+# stream definitions define connections between spouts and bolts.
+# note that such connections can be cyclical
+streams:
+ - name: "spout-1 --> bolt-1" # name isn't used (placeholder for logging, UI, etc.)
+# id: "connection-1"
+ from: "spout-1"
+ to: "bolt-1"
+ grouping:
+ type: FIELDS
+ args: ["word"]
+
+ - name: "bolt-1 --> bolt2"
+ from: "bolt-1"
+ to: "bolt-2"
+ grouping:
+ type: SHUFFLE
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-elasticsearch-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-elasticsearch-examples/pom.xml b/examples/storm-elasticsearch-examples/pom.xml
index 0c312fc..f853ec4 100644
--- a/examples/storm-elasticsearch-examples/pom.xml
+++ b/examples/storm-elasticsearch-examples/pom.xml
@@ -26,21 +26,6 @@
</parent>
<artifactId>storm-elasticsearch-examples</artifactId>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <provided.scope>provided</provided.scope>
- </properties>
-
- <profiles>
- <profile>
- <id>intellij</id>
- <properties>
- <provided.scope>compile</provided.scope>
- </properties>
- </profile>
- </profiles>
-
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-hbase-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/pom.xml b/examples/storm-hbase-examples/pom.xml
index 1b26e92..8b1c15c 100644
--- a/examples/storm-hbase-examples/pom.xml
+++ b/examples/storm-hbase-examples/pom.xml
@@ -27,20 +27,6 @@
<artifactId>storm-hbase-examples</artifactId>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <provided.scope>provided</provided.scope>
- </properties>
-
- <profiles>
- <profile>
- <id>intellij</id>
- <properties>
- <provided.scope>compile</provided.scope>
- </properties>
- </profile>
- </profiles>
-
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-hdfs-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/pom.xml b/examples/storm-hdfs-examples/pom.xml
index 78ac00a..69b90ef 100644
--- a/examples/storm-hdfs-examples/pom.xml
+++ b/examples/storm-hdfs-examples/pom.xml
@@ -27,20 +27,6 @@
<artifactId>storm-hdfs-examples</artifactId>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <provided.scope>provided</provided.scope>
- </properties>
-
- <profiles>
- <profile>
- <id>intellij</id>
- <properties>
- <provided.scope>compile</provided.scope>
- </properties>
- </profile>
- </profiles>
-
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-hive-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-hive-examples/pom.xml b/examples/storm-hive-examples/pom.xml
index a51f597..6f3f6ba 100644
--- a/examples/storm-hive-examples/pom.xml
+++ b/examples/storm-hive-examples/pom.xml
@@ -27,20 +27,6 @@
<artifactId>storm-hive-examples</artifactId>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <provided.scope>provided</provided.scope>
- </properties>
-
- <profiles>
- <profile>
- <id>intellij</id>
- <properties>
- <provided.scope>compile</provided.scope>
- </properties>
- </profile>
- </profiles>
-
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-jdbc-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-jdbc-examples/pom.xml b/examples/storm-jdbc-examples/pom.xml
index fa2cae9..9d488b2 100644
--- a/examples/storm-jdbc-examples/pom.xml
+++ b/examples/storm-jdbc-examples/pom.xml
@@ -27,20 +27,6 @@
<artifactId>storm-jdbc-examples</artifactId>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <provided.scope>provided</provided.scope>
- </properties>
-
- <profiles>
- <profile>
- <id>intellij</id>
- <properties>
- <provided.scope>compile</provided.scope>
- </properties>
- </profile>
- </profiles>
-
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-jms-examples/README.markdown
----------------------------------------------------------------------
diff --git a/examples/storm-jms-examples/README.markdown b/examples/storm-jms-examples/README.markdown
new file mode 100644
index 0000000..7a4d8f0
--- /dev/null
+++ b/examples/storm-jms-examples/README.markdown
@@ -0,0 +1,12 @@
+## About Storm JMS Examples
+This project contains a simple storm topology that illustrates the usage of "storm-jms".
+
+To build:
+
+`mvn clean install`
+
+The default build will create a jar file that can be deployed to to a Storm cluster in the "target" directory:
+
+`storm-jms-examples-0.1-SNAPSHOT-jar-with-dependencies.jar`
+
+
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-jms-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-jms-examples/pom.xml b/examples/storm-jms-examples/pom.xml
new file mode 100644
index 0000000..4d232f7
--- /dev/null
+++ b/examples/storm-jms-examples/pom.xml
@@ -0,0 +1,150 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>storm</artifactId>
+ <groupId>org.apache.storm</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+
+ <artifactId>storm-jms-examples</artifactId>
+
+ <properties>
+ <spring.version>2.5.6</spring.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${spring.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>${spring.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${spring.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jms</artifactId>
+ <version>${spring.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.xbean</groupId>
+ <artifactId>xbean-spring</artifactId>
+ <version>3.7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${project.version}</version>
+ <!-- keep storm out of the jar-with-dependencies -->
+ <scope>${provided.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-jms</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.activemq</groupId>
+ <artifactId>activemq-core</artifactId>
+ <version>5.4.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <!-- bind the maven-assembly-plugin to the package phase this will create
+ a jar file without the storm dependencies suitable for deployment to a cluster. -->
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <archive>
+ <manifest>
+ </manifest>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>java</executable>
+ <includeProjectDependencies>true</includeProjectDependencies>
+ <includePluginDependencies>true</includePluginDependencies>
+ <mainClass>org.apache.storm.jms.example.ExampleJmsTopology</mainClass>
+ <systemProperties>
+ <systemProperty>
+ <key>log4j.configuration</key>
+ <value>file:./src/main/resources/log4j.properties</value>
+ </systemProperty>
+ </systemProperties>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ </dependency>
+ </dependencies>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
new file mode 100644
index 0000000..82dbd5b
--- /dev/null
+++ b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.jms.example;
+
+import javax.jms.JMSException;
+import javax.jms.Message;
+import javax.jms.Session;
+import javax.jms.TextMessage;
+
+import org.apache.storm.Config;
+import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
+import org.apache.storm.StormSubmitter;
+import org.apache.storm.jms.JmsMessageProducer;
+import org.apache.storm.jms.JmsProvider;
+import org.apache.storm.jms.JmsTupleProducer;
+import org.apache.storm.jms.bolt.JmsBolt;
+import org.apache.storm.jms.spout.JmsSpout;
+import org.apache.storm.topology.TopologyBuilder;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.ITuple;
+import org.apache.storm.utils.Utils;
+
+public class ExampleJmsTopology {
+ public static final String JMS_QUEUE_SPOUT = "JMS_QUEUE_SPOUT";
+ public static final String INTERMEDIATE_BOLT = "INTERMEDIATE_BOLT";
+ public static final String FINAL_BOLT = "FINAL_BOLT";
+ public static final String JMS_TOPIC_BOLT = "JMS_TOPIC_BOLT";
+ public static final String JMS_TOPIC_SPOUT = "JMS_TOPIC_SPOUT";
+ public static final String ANOTHER_BOLT = "ANOTHER_BOLT";
+
+ @SuppressWarnings("serial")
+ public static void main(String[] args) throws Exception {
+
+ // JMS Queue Provider
+ JmsProvider jmsQueueProvider = new SpringJmsProvider(
+ "jms-activemq.xml", "jmsConnectionFactory",
+ "notificationQueue");
+
+ // JMS Topic provider
+ JmsProvider jmsTopicProvider = new SpringJmsProvider(
+ "jms-activemq.xml", "jmsConnectionFactory",
+ "notificationTopic");
+
+ // JMS Producer
+ JmsTupleProducer producer = new JsonTupleProducer();
+
+ // JMS Queue Spout
+ JmsSpout queueSpout = new JmsSpout();
+ queueSpout.setJmsProvider(jmsQueueProvider);
+ queueSpout.setJmsTupleProducer(producer);
+ queueSpout.setJmsAcknowledgeMode(Session.CLIENT_ACKNOWLEDGE);
+ queueSpout.setDistributed(true); // allow multiple instances
+
+ TopologyBuilder builder = new TopologyBuilder();
+
+ // spout with 5 parallel instances
+ builder.setSpout(JMS_QUEUE_SPOUT, queueSpout, 5);
+
+ // intermediate bolt, subscribes to jms spout, anchors on tuples, and auto-acks
+ builder.setBolt(INTERMEDIATE_BOLT,
+ new GenericBolt("INTERMEDIATE_BOLT", true, true, new Fields("json")), 3).shuffleGrouping(
+ JMS_QUEUE_SPOUT);
+
+ // bolt that subscribes to the intermediate bolt, and auto-acks
+ // messages.
+ builder.setBolt(FINAL_BOLT, new GenericBolt("FINAL_BOLT", true, true), 3).shuffleGrouping(
+ INTERMEDIATE_BOLT);
+
+ // bolt that subscribes to the intermediate bolt, and publishes to a JMS Topic
+ JmsBolt jmsBolt = new JmsBolt();
+ jmsBolt.setJmsProvider(jmsTopicProvider);
+
+ // anonymous message producer just calls toString() on the tuple to create a jms message
+ jmsBolt.setJmsMessageProducer(new JmsMessageProducer() {
+ @Override
+ public Message toMessage(Session session, ITuple input) throws JMSException {
+ System.out.println("Sending JMS Message:" + input.toString());
+ TextMessage tm = session.createTextMessage(input.toString());
+ return tm;
+ }
+ });
+
+ builder.setBolt(JMS_TOPIC_BOLT, jmsBolt).shuffleGrouping(INTERMEDIATE_BOLT);
+
+ // JMS Topic spout
+ JmsSpout topicSpout = new JmsSpout();
+ topicSpout.setJmsProvider(jmsTopicProvider);
+ topicSpout.setJmsTupleProducer(producer);
+ topicSpout.setJmsAcknowledgeMode(Session.CLIENT_ACKNOWLEDGE);
+ topicSpout.setDistributed(false);
+
+ builder.setSpout(JMS_TOPIC_SPOUT, topicSpout);
+
+ builder.setBolt(ANOTHER_BOLT, new GenericBolt("ANOTHER_BOLT", true, true), 1).shuffleGrouping(
+ JMS_TOPIC_SPOUT);
+
+ Config conf = new Config();
+
+ if (args.length > 0) {
+ conf.setNumWorkers(3);
+
+ StormSubmitter.submitTopology(args[0], conf,
+ builder.createTopology());
+ } else {
+
+ conf.setDebug(true);
+
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("storm-jms-example", conf, builder.createTopology());) {
+ Utils.sleep(60000);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/GenericBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/GenericBolt.java b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/GenericBolt.java
new file mode 100644
index 0000000..57de1ba
--- /dev/null
+++ b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/GenericBolt.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.jms.example;
+
+import java.util.Map;
+
+import org.apache.storm.topology.base.BaseRichBolt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+
+/**
+ * A generic <code>org.apache.storm.topology.IRichBolt</code> implementation
+ * for testing/debugging the Storm JMS Spout and example topologies.
+ * <p>
+ * For debugging purposes, set the log level of the
+ * <code>org.apache.storm.contrib.jms</code> package to DEBUG for debugging
+ * output.
+ *
+ * @author tgoetz
+ */
+@SuppressWarnings("serial")
+public class GenericBolt extends BaseRichBolt {
+ private static final Logger LOG = LoggerFactory.getLogger(GenericBolt.class);
+ private OutputCollector collector;
+ private boolean autoAck = false;
+ private boolean autoAnchor = false;
+ private Fields declaredFields;
+ private String name;
+
+ /**
+ * Constructs a new <code>GenericBolt</code> instance.
+ *
+ * @param name The name of the bolt (used in DEBUG logging)
+ * @param autoAck Whether or not this bolt should automatically acknowledge received tuples.
+ * @param autoAnchor Whether or not this bolt should automatically anchor to received tuples.
+ * @param declaredFields The fields this bolt declares as output.
+ */
+ public GenericBolt(String name, boolean autoAck, boolean autoAnchor, Fields declaredFields) {
+ this.name = name;
+ this.autoAck = autoAck;
+ this.autoAnchor = autoAnchor;
+ this.declaredFields = declaredFields;
+ }
+
+ public GenericBolt(String name, boolean autoAck, boolean autoAnchor) {
+ this(name, autoAck, autoAnchor, null);
+ }
+
+ @SuppressWarnings("rawtypes")
+ public void prepare(Map stormConf, TopologyContext context,
+ OutputCollector collector) {
+ this.collector = collector;
+
+ }
+
+ public void execute(Tuple input) {
+ LOG.debug("[" + this.name + "] Received message: " + input);
+
+
+ // only emit if we have declared fields.
+ if (this.declaredFields != null) {
+ LOG.debug("[" + this.name + "] emitting: " + input);
+ if (this.autoAnchor) {
+ this.collector.emit(input, input.getValues());
+ } else {
+ this.collector.emit(input.getValues());
+ }
+ }
+
+ if (this.autoAck) {
+ LOG.debug("[" + this.name + "] ACKing tuple: " + input);
+ this.collector.ack(input);
+ }
+
+ }
+
+ public void cleanup() {
+
+ }
+
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ if (this.declaredFields != null) {
+ declarer.declare(this.declaredFields);
+ }
+ }
+
+ public boolean isAutoAck() {
+ return this.autoAck;
+ }
+
+ public void setAutoAck(boolean autoAck) {
+ this.autoAck = autoAck;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/2a369e40/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/JsonTupleProducer.java
----------------------------------------------------------------------
diff --git a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/JsonTupleProducer.java b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/JsonTupleProducer.java
new file mode 100644
index 0000000..9ee175e
--- /dev/null
+++ b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/JsonTupleProducer.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.jms.example;
+
+import javax.jms.JMSException;
+import javax.jms.Message;
+import javax.jms.TextMessage;
+
+import org.apache.storm.jms.JmsTupleProducer;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
+
+/**
+ * A simple <code>JmsTupleProducer</code> that expects to receive
+ * JMS <code>TextMessage</code> objects with a body in JSON format.
+ * <p/>
+ * Ouputs a tuple with field name "json" and a string value
+ * containing the raw json.
+ * <p/>
+ * <b>NOTE: </b> Currently this implementation assumes the text is valid
+ * JSON and does not attempt to parse or validate it.
+ *
+ * @author tgoetz
+ *
+ */
+@SuppressWarnings("serial")
+public class JsonTupleProducer implements JmsTupleProducer {
+
+ public Values toTuple(Message msg) throws JMSException {
+ if(msg instanceof TextMessage){
+ String json = ((TextMessage) msg).getText();
+ return new Values(json);
+ } else {
+ return null;
+ }
+ }
+
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("json"));
+ }
+
+}