You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by mi...@apache.org on 2014/06/24 17:02:18 UTC
[1/2] git commit: storm-starter: add support to RollingTopWords to
run on a real Storm cluster
Repository: incubator-storm
Updated Branches:
refs/heads/master 9b6c0f656 -> 4c70e407c
storm-starter: add support to RollingTopWords to run on a real Storm cluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/a80f95e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/a80f95e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/a80f95e4
Branch: refs/heads/master
Commit: a80f95e45c202fb14454f4330730970ec75d13fa
Parents: 9b6c0f6
Author: Michael G. Noll <mi...@apache.org>
Authored: Tue Jun 24 16:03:51 2014 +0200
Committer: Michael G. Noll <mi...@apache.org>
Committed: Tue Jun 24 16:46:43 2014 +0200
----------------------------------------------------------------------
.../src/jvm/storm/starter/RollingTopWords.java | 62 ++++++++++++++++++--
.../src/jvm/storm/starter/util/StormRunner.java | 8 +++
2 files changed, 66 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/a80f95e4/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java b/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java
index 2630557..515008f 100644
--- a/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java
+++ b/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java
@@ -21,6 +21,7 @@ import backtype.storm.Config;
import backtype.storm.testing.TestWordSpout;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;
+import org.apache.log4j.Logger;
import storm.starter.bolt.IntermediateRankingsBolt;
import storm.starter.bolt.RollingCountBolt;
import storm.starter.bolt.TotalRankingsBolt;
@@ -33,6 +34,7 @@ import storm.starter.util.StormRunner;
*/
public class RollingTopWords {
+ private static final Logger LOG = Logger.getLogger(RollingTopWords.class);
private static final int DEFAULT_RUNTIME_IN_SECONDS = 60;
private static final int TOP_N = 5;
@@ -41,9 +43,9 @@ public class RollingTopWords {
private final Config topologyConfig;
private final int runtimeInSeconds;
- public RollingTopWords() throws InterruptedException {
+ public RollingTopWords(String topologyName) throws InterruptedException {
builder = new TopologyBuilder();
- topologyName = "slidingWindowCounts";
+ this.topologyName = topologyName;
topologyConfig = createTopologyConfiguration();
runtimeInSeconds = DEFAULT_RUNTIME_IN_SECONDS;
@@ -68,11 +70,63 @@ public class RollingTopWords {
builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
}
- public void run() throws InterruptedException {
+ public void runLocally() throws InterruptedException {
StormRunner.runTopologyLocally(builder.createTopology(), topologyName, topologyConfig, runtimeInSeconds);
}
+ public void runRemotely() throws Exception {
+ StormRunner.runTopologyRemotely(builder.createTopology(), topologyName, topologyConfig);
+ }
+
+ /**
+ * Submits (runs) the topology.
+ *
+ * Usage: "RollingTopWords [topology-name] [local|remote]"
+ *
+ * By default, the topology is run locally under the name "slidingWindowCounts".
+ *
+ * Examples:
+ *
+ * <pre>
+ * {@code
+ *
+ * # Runs in local mode (LocalCluster), with topology name "slidingWindowCounts"
+ * $ storm jar storm-starter-jar-with-dependencies.jar storm.starter.RollingTopWords
+ *
+ * # Runs in local mode (LocalCluster), with topology name "foobar"
+ * $ storm jar storm-starter-jar-with-dependencies.jar storm.starter.RollingTopWords foobar
+ *
+ * # Runs in local mode (LocalCluster), with topology name "foobar"
+ * $ storm jar storm-starter-jar-with-dependencies.jar storm.starter.RollingTopWords foobar local
+ *
+ * # Runs in remote/cluster mode, with topology name "production-topology"
+ * $ storm jar storm-starter-jar-with-dependencies.jar storm.starter.RollingTopWords production-topology remote
+ * }
+ * </pre>
+ *
+ * @param args First positional argument (optional) is topology name, second positional argument (optional) defines
+ * whether to run the topology locally ("local") or remotely, i.e. on a real cluster ("remote").
+ * @throws Exception
+ */
public static void main(String[] args) throws Exception {
- new RollingTopWords().run();
+ String topologyName = "slidingWindowCounts";
+ if (args.length >= 1) {
+ topologyName = args[0];
+ }
+ boolean runLocally = true;
+ if (args.length >= 2 && args[1].equalsIgnoreCase("remote")) {
+ runLocally = false;
+ }
+
+ LOG.info("Topology name: " + topologyName);
+ RollingTopWords rtw = new RollingTopWords(topologyName);
+ if (runLocally) {
+ LOG.info("Running in local mode");
+ rtw.runLocally();
+ }
+ else {
+ LOG.info("Running in remote (cluster) mode");
+ rtw.runRemotely();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/a80f95e4/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java b/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java
index f916ec6..f3017ce 100644
--- a/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java
+++ b/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java
@@ -19,6 +19,9 @@ package storm.starter.util;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
+import backtype.storm.StormSubmitter;
+import backtype.storm.generated.AlreadyAliveException;
+import backtype.storm.generated.InvalidTopologyException;
import backtype.storm.generated.StormTopology;
public final class StormRunner {
@@ -36,4 +39,9 @@ public final class StormRunner {
cluster.killTopology(topologyName);
cluster.shutdown();
}
+
+ public static void runTopologyRemotely(StormTopology topology, String topologyName, Config conf)
+ throws AlreadyAliveException, InvalidTopologyException {
+ StormSubmitter.submitTopology(topologyName, conf, topology);
+ }
}
[2/2] git commit: storm-starter: explain how to run a topology from a
fat jar
Posted by mi...@apache.org.
storm-starter: explain how to run a topology from a fat jar
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/4c70e407
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/4c70e407
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/4c70e407
Branch: refs/heads/master
Commit: 4c70e407c33b8cb5f770f22296851c6b1ff6d7c9
Parents: a80f95e
Author: Michael G. Noll <mi...@apache.org>
Authored: Tue Jun 24 17:02:11 2014 +0200
Committer: Michael G. Noll <mi...@apache.org>
Committed: Tue Jun 24 17:02:11 2014 +0200
----------------------------------------------------------------------
examples/storm-starter/README.markdown | 27 ++++++++++++++++++++++++++-
1 file changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/4c70e407/examples/storm-starter/README.markdown
----------------------------------------------------------------------
diff --git a/examples/storm-starter/README.markdown b/examples/storm-starter/README.markdown
index 98a9749..283f85d 100644
--- a/examples/storm-starter/README.markdown
+++ b/examples/storm-starter/README.markdown
@@ -93,9 +93,34 @@ You can package a jar suitable for submitting to a Storm cluster with the comman
$ mvn package
-This will package your code and all the non-Storm dependencies into a single "uberjar" at the path
+This will package your code and all the non-Storm dependencies into a single "uberjar" (or "fat jar") at the path
`target/storm-starter-{version}-jar-with-dependencies.jar`.
+Example filename of the uberjar:
+
+ >>> target/storm-starter-0.9.3-incubating-SNAPSHOT-jar-with-dependencies.jar
+
+You can submit (run) a topology contained in this uberjar to Storm via the `storm` CLI tool:
+
+ # Example 1: Run the RollingTopWords in local mode (LocalCluster)
+ $ storm jar storm-starter-*-jar-with-dependencies.jar storm.starter.RollingTopWords
+
+ # Example 2: Run the RollingTopWords in remote/cluster mode,
+ # under the name "production-topology"
+ $ storm jar storm-starter-*-jar-with-dependencies.jar storm.starter.RollingTopWords production-topology remote
+
+_Submitting a topology in local vs. remote mode:_
+It depends on the actual code of a topology how you can or even must tell Storm whether to run the topology locally (in
+an in-memory LocalCluster instance of Storm) or remotely (in a "real" Storm cluster). In the case of
+[RollingTopWords](src/jvm/storm/starter/RollingTopWords.java), for instance, this can be done by passing command line
+arguments.
+Topologies other than `RollingTopWords` -- such as [ExclamationTopology](src/jvm/storm/starter/ExclamationTopology.java)
+-- may behave differently, e.g. by always submitting to a remote cluster (i.e. hardcoded in a way that you, as a user,
+cannot change without modifying the topology code), or by requiring a customized configuration file that the topology
+code will parse prior submitting the topology to Storm. Similarly, further options such as the name of the topology may
+be user-configurable or be hardcoded into the topology code. So make sure you understand how the topology of your
+choice is set up and configured!
+
## Running unit tests