You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/04/22 18:39:13 UTC
[01/10] git commit: Do not rely on spaces to separate command args
Repository: incubator-storm
Updated Branches:
refs/heads/master 8f3949534 -> 93612f565
Do not rely on spaces to separate command args
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/47387185
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/47387185
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/47387185
Branch: refs/heads/master
Commit: 47387185b1c8b61b24c8c3be1675e4e4f2c670c6
Parents: ea0f54e
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Sun Jan 12 00:48:52 2014 +0000
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Wed Jan 15 03:16:51 2014 +0000
----------------------------------------------------------------------
.../clj/backtype/storm/daemon/supervisor.clj | 38 ++++++++++++--------
storm-core/src/clj/backtype/storm/util.clj | 4 +--
2 files changed, 24 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/47387185/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 43cb6fe..9bff2ad 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -447,22 +447,30 @@
stormjar (supervisor-stormjar-path stormroot)
storm-conf (read-supervisor-storm-conf conf storm-id)
classpath (add-to-classpath (current-classpath) [stormjar])
- childopts (.replaceAll (str (conf WORKER-CHILDOPTS) " " (storm-conf TOPOLOGY-WORKER-CHILDOPTS))
- "%ID%"
- (str port))
+ worker-childopts (when-let [s (conf WORKER-CHILDOPTS)]
+ (.replaceAll s "%ID%" (str port)))
+ topo-worker-childopts (when-let [s (conf TOPOLOGY-WORKER-CHILDOPTS)]
+ (.replaceAll s "%ID%" (str port)))
logfilename (str "worker-" port ".log")
- command (str "java -server " childopts
- " -Djava.library.path=" (conf JAVA-LIBRARY-PATH)
- " -Dlogfile.name=" logfilename
- " -Dstorm.home=" storm-home
- " -Dlogback.configurationFile=" storm-home "/logback/cluster.xml"
- " -Dstorm.id=" storm-id
- " -Dworker.id=" worker-id
- " -Dworker.port=" port
- " -cp " classpath " backtype.storm.daemon.worker "
- (java.net.URLEncoder/encode storm-id) " " (:assignment-id supervisor)
- " " port " " worker-id)]
- (log-message "Launching worker with command: " command)
+ command ["java"
+ "-server"
+ worker-childopts
+ topo-worker-childopts
+ (str "-Djava.library.path=" (conf JAVA-LIBRARY-PATH))
+ (str "-Dlogfile.name=" logfilename)
+ (str "-Dstorm.home=" storm-home)
+ (str "-Dlogback.configurationFile=" storm-home "/logback/cluster.xml")
+ (str "-Dstorm.id=" storm-id)
+ (str "-Dworker.id=" worker-id)
+ (str "-Dworker.port=" port)
+ "-cp" classpath
+ "backtype.storm.daemon.worker"
+ (java.net.URLEncoder/encode storm-id)
+ (:assignment-id supervisor)
+ port
+ worker-id]
+ command (->> command (map str) (filter (complement empty?)))]
+ (log-message "Launching worker with command: " (clojure.string/join " " command))
(launch-process command :environment {"LD_LIBRARY_PATH" (conf JAVA-LIBRARY-PATH)})
))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/47387185/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj
index 2ab30c7..a3d5ebb 100644
--- a/storm-core/src/clj/backtype/storm/util.clj
+++ b/storm-core/src/clj/backtype/storm/util.clj
@@ -395,9 +395,7 @@
))
(defnk launch-process [command :environment {}]
- (let [command (->> (seq (.split command " "))
- (filter (complement empty?)))
- builder (ProcessBuilder. command)
+ (let [builder (ProcessBuilder. command)
process-env (.environment builder)]
(doseq [[k v] environment]
(.put process-env k v))
[07/10] git commit: Merge remote-tracking branch 'apache/master' into
d2r-worker-lancher-args
Posted by bo...@apache.org.
Merge remote-tracking branch 'apache/master' into d2r-worker-lancher-args
Conflicts:
pom.xml
storm-core/src/clj/backtype/storm/ui/core.clj
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/bf557f89
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/bf557f89
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/bf557f89
Branch: refs/heads/master
Commit: bf557f892702551f33733b95c29c431df5c9816c
Parents: 2fb66fa 1a0b46e
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Tue Apr 15 13:33:20 2014 +0000
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Tue Apr 15 13:33:20 2014 +0000
----------------------------------------------------------------------
.gitignore | 2 +-
CHANGELOG.md | 29 +
DISCLAIMER | 10 +
NOTICE | 9 +-
README.markdown | 5 +-
bin/storm | 19 +-
bin/storm-config.cmd | 19 +-
bin/storm.cmd | 14 +-
examples/storm-starter/README.markdown | 108 +
.../multilang/resources/splitsentence.py | 24 +
.../multilang/resources/splitsentence.rb | 26 +
.../storm-starter/multilang/resources/storm.py | 221 ++
.../storm-starter/multilang/resources/storm.rb | 200 ++
examples/storm-starter/pom.xml | 161 ++
.../src/clj/storm/starter/clj/word_count.clj | 95 +
.../jvm/storm/starter/BasicDRPCTopology.java | 78 +
.../jvm/storm/starter/ExclamationTopology.java | 87 +
.../src/jvm/storm/starter/ManualDRPC.java | 68 +
.../jvm/storm/starter/PrintSampleStream.java | 54 +
.../src/jvm/storm/starter/ReachTopology.java | 196 ++
.../src/jvm/storm/starter/RollingTopWords.java | 78 +
.../jvm/storm/starter/SingleJoinExample.java | 64 +
.../storm/starter/TransactionalGlobalCount.java | 173 ++
.../jvm/storm/starter/TransactionalWords.java | 246 ++
.../jvm/storm/starter/WordCountTopology.java | 107 +
.../storm/starter/bolt/AbstractRankerBolt.java | 110 +
.../starter/bolt/IntermediateRankingsBolt.java | 58 +
.../src/jvm/storm/starter/bolt/PrinterBolt.java | 37 +
.../storm/starter/bolt/RollingCountBolt.java | 142 ++
.../jvm/storm/starter/bolt/SingleJoinBolt.java | 114 +
.../storm/starter/bolt/TotalRankingsBolt.java | 59 +
.../starter/spout/RandomSentenceSpout.java | 64 +
.../storm/starter/spout/TwitterSampleSpout.java | 122 +
.../tools/NthLastModifiedTimeTracker.java | 70 +
.../src/jvm/storm/starter/tools/Rankable.java | 32 +
.../starter/tools/RankableObjectWithFields.java | 148 ++
.../src/jvm/storm/starter/tools/Rankings.java | 156 ++
.../starter/tools/SlidingWindowCounter.java | 119 +
.../storm/starter/tools/SlotBasedCounter.java | 118 +
.../jvm/storm/starter/trident/TridentReach.java | 156 ++
.../storm/starter/trident/TridentWordCount.java | 85 +
.../src/jvm/storm/starter/util/StormRunner.java | 39 +
.../jvm/storm/starter/util/TupleHelpers.java | 33 +
.../bolt/IntermediateRankingsBoltTest.java | 146 ++
.../starter/bolt/RollingCountBoltTest.java | 113 +
.../starter/bolt/TotalRankingsBoltTest.java | 147 ++
.../storm/starter/tools/MockTupleHelpers.java | 40 +
.../tools/NthLastModifiedTimeTrackerTest.java | 125 +
.../tools/RankableObjectWithFieldsTest.java | 252 ++
.../jvm/storm/starter/tools/RankingsTest.java | 368 +++
.../starter/tools/SlidingWindowCounterTest.java | 106 +
.../starter/tools/SlotBasedCounterTest.java | 181 ++
pom.xml | 121 +-
.../maven-shade-clojure-transformer/pom.xml | 39 +
.../maven/shade/clojure/ClojureTransformer.java | 72 +
storm-core/pom.xml | 77 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 8 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 8 +-
storm-core/src/clj/backtype/storm/thrift.clj | 4 +-
storm-core/src/clj/backtype/storm/timer.clj | 2 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 28 +-
storm-core/src/clj/backtype/storm/util.clj | 5 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 4 +-
.../src/jvm/backtype/storm/StormSubmitter.java | 6 +-
.../storm/drpc/DRPCInvocationsClient.java | 10 +-
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 4 +-
.../jvm/backtype/storm/drpc/ReturnResults.java | 2 +-
.../storm/generated/AlreadyAliveException.java | 46 +-
.../src/jvm/backtype/storm/generated/Bolt.java | 60 +-
.../jvm/backtype/storm/generated/BoltStats.java | 182 +-
.../storm/generated/ClusterSummary.java | 86 +-
.../storm/generated/ComponentCommon.java | 100 +-
.../storm/generated/ComponentObject.java | 66 +-
.../storm/generated/DRPCExecutionException.java | 46 +-
.../backtype/storm/generated/DRPCRequest.java | 60 +-
.../storm/generated/DistributedRPC.java | 180 +-
.../generated/DistributedRPCInvocations.java | 384 +--
.../jvm/backtype/storm/generated/ErrorInfo.java | 60 +-
.../backtype/storm/generated/ExecutorInfo.java | 60 +-
.../storm/generated/ExecutorSpecificStats.java | 56 +-
.../backtype/storm/generated/ExecutorStats.java | 106 +-
.../storm/generated/ExecutorSummary.java | 114 +-
.../storm/generated/GlobalStreamId.java | 60 +-
.../jvm/backtype/storm/generated/Grouping.java | 112 +-
.../generated/InvalidTopologyException.java | 46 +-
.../backtype/storm/generated/JavaObject.java | 66 +-
.../backtype/storm/generated/JavaObjectArg.java | 90 +-
.../backtype/storm/generated/KillOptions.java | 44 +-
.../jvm/backtype/storm/generated/Nimbus.java | 2368 +++++++++---------
.../storm/generated/NotAliveException.java | 46 +-
.../backtype/storm/generated/NullStruct.java | 32 +-
.../storm/generated/RebalanceOptions.java | 76 +-
.../storm/generated/ShellComponent.java | 56 +-
.../jvm/backtype/storm/generated/SpoutSpec.java | 60 +-
.../backtype/storm/generated/SpoutStats.java | 122 +-
.../storm/generated/StateSpoutSpec.java | 60 +-
.../backtype/storm/generated/StormTopology.java | 98 +-
.../backtype/storm/generated/StreamInfo.java | 66 +-
.../backtype/storm/generated/SubmitOptions.java | 46 +-
.../storm/generated/SupervisorSummary.java | 102 +-
.../backtype/storm/generated/TopologyInfo.java | 136 +-
.../storm/generated/TopologyInitialStatus.java | 4 +-
.../storm/generated/TopologySummary.java | 130 +-
.../backtype/storm/messaging/netty/Client.java | 195 +-
.../backtype/storm/messaging/netty/Context.java | 30 +-
.../storm/messaging/netty/MessageBatch.java | 55 +-
.../backtype/storm/messaging/netty/Server.java | 2 +-
.../messaging/netty/StormClientHandler.java | 80 +-
.../storm/security/auth/ITransportPlugin.java | 8 +-
.../security/auth/SaslTransportPlugin.java | 24 +-
.../security/auth/SimpleTransportPlugin.java | 24 +-
.../storm/security/auth/ThriftClient.java | 10 +-
.../storm/security/auth/ThriftServer.java | 4 +-
.../auth/digest/DigestSaslTransportPlugin.java | 10 +-
.../jvm/backtype/storm/utils/DRPCClient.java | 10 +-
.../jvm/backtype/storm/utils/NimbusClient.java | 2 +-
.../src/jvm/backtype/storm/utils/Time.java | 18 +-
.../src/jvm/backtype/storm/utils/Utils.java | 37 +-
.../trident/drpc/ReturnResultsReducer.java | 2 +-
.../jvm/storm/trident/state/map/OpaqueMap.java | 6 +-
.../trident/state/map/RemovableMapState.java | 8 +
.../storm/trident/testing/MemoryMapState.java | 27 +-
.../jvm/storm/trident/util/TridentUtils.java | 8 +-
.../storm/security/auth/ThriftClient_test.clj | 2 +-
.../storm/security/auth/ThriftServer_test.clj | 2 +-
.../backtype/storm/security/auth/auth_test.clj | 4 +-
.../test/clj/backtype/storm/utils_test.clj | 13 +-
.../test/clj/storm/trident/state_test.clj | 33 +-
storm-deps/libthrift/pom.xml | 87 -
storm-dist/LICENSE | 297 ---
storm-dist/NOTICE | 36 -
storm-dist/binary/LICENSE | 297 +++
storm-dist/binary/NOTICE | 31 +
storm-dist/binary/pom.xml | 78 +
storm-dist/binary/src/main/assembly/binary.xml | 117 +
storm-dist/pom.xml | 90 -
storm-dist/source/pom.xml | 77 +
storm-dist/source/src/main/assembly/source.xml | 42 +
storm-dist/src/main/assembly/binary.xml | 100 -
storm-dist/src/main/assembly/source.xml | 42 -
140 files changed, 9210 insertions(+), 3747 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/bf557f89/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index 3a0c26b,8894aaf..4b35ab4
--- a/pom.xml
+++ b/pom.xml
@@@ -166,8 -176,7 +176,8 @@@
<clojure-complete.version>0.2.3</clojure-complete.version>
<mockito.version>1.9.5</mockito.version>
<reply.version>0.3.0</reply.version>
+ <conjure.version>2.1.3</conjure.version>
-
+ <zookeeper.version>3.4.5</zookeeper.version>
</properties>
@@@ -412,23 -444,18 +445,24 @@@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.clojars.runa</groupId>
+ <artifactId>conjure</artifactId>
+ <version>${conjure.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>libthrift7</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>0.7.0</version>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
</dependency>
+ <!-- used by examples/storm-starter -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.1</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/bf557f89/storm-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/bf557f89/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index e46871a,ad4b97f..87e79fa
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -1017,9 -1016,10 +1017,10 @@@
(-> (main-page)
ui-template))
(GET "/topology/:id" [:as {cookies :cookies} id & m]
- (let [include-sys? (get-include-sys? cookies)]
+ (let [include-sys? (get-include-sys? cookies)
+ id (java.net.URLDecoder/decode id)]
(try
- (-> (topology-page id (:window m) include-sys?)
+ (-> (topology-page (URLDecoder/decode id) (:window m) include-sys?)
(concat [(mk-system-toggle-button include-sys?)])
ui-template)
(catch Exception e (resp/redirect "/")))))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/bf557f89/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
[05/10] git commit: Support *.worker.childopts as list or as string
Posted by bo...@apache.org.
Support *.worker.childopts as list or as string
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/c970225d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/c970225d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/c970225d
Branch: refs/heads/master
Commit: c970225dc43721e1a7827408df7542aed94e11cb
Parents: 6d1ad6a
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Wed Jan 15 02:02:41 2014 +0000
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Wed Jan 15 03:16:53 2014 +0000
----------------------------------------------------------------------
.../clj/backtype/storm/daemon/supervisor.clj | 45 +++++++++++---------
storm-core/src/jvm/backtype/storm/Config.java | 5 +--
.../jvm/backtype/storm/ConfigValidation.java | 17 ++++++++
.../test/clj/backtype/storm/config_test.clj | 20 +++++++++
4 files changed, 64 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/c970225d/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index a38ee5c..1630766 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -438,6 +438,11 @@
(FileUtils/moveDirectory (File. tmproot) (File. stormroot))
))
+(defn- substitute-worker-childopts [value port]
+ (let [sub-fn (fn [s] (.replaceAll s "%ID%" (str port)))]
+ (if (list? value)
+ (map sub-fn value)
+ (-> value sub-fn (.split " ")))))
(defmethod launch-worker
:distributed [supervisor storm-id port worker-id]
@@ -448,27 +453,27 @@
storm-conf (read-supervisor-storm-conf conf storm-id)
classpath (add-to-classpath (current-classpath) [stormjar])
worker-childopts (when-let [s (conf WORKER-CHILDOPTS)]
- (.replaceAll s "%ID%" (str port)))
- topo-worker-childopts (when-let [s (conf TOPOLOGY-WORKER-CHILDOPTS)]
- (.replaceAll s "%ID%" (str port)))
+ (substitute-worker-childopts s port))
+ topo-worker-childopts (when-let [s (storm-conf TOPOLOGY-WORKER-CHILDOPTS)]
+ (substitute-worker-childopts s port))
logfilename (str "worker-" port ".log")
- command ["java"
- "-server"
- worker-childopts
- topo-worker-childopts
- (str "-Djava.library.path=" (conf JAVA-LIBRARY-PATH))
- (str "-Dlogfile.name=" logfilename)
- (str "-Dstorm.home=" storm-home)
- (str "-Dlogback.configurationFile=" storm-home "/logback/cluster.xml")
- (str "-Dstorm.id=" storm-id)
- (str "-Dworker.id=" worker-id)
- (str "-Dworker.port=" port)
- "-cp" classpath
- "backtype.storm.daemon.worker"
- storm-id
- (:assignment-id supervisor)
- port
- worker-id]
+ command (concat
+ ["java" "-server"]
+ worker-childopts
+ topo-worker-childopts
+ [(str "-Djava.library.path=" (conf JAVA-LIBRARY-PATH))
+ (str "-Dlogfile.name=" logfilename)
+ (str "-Dstorm.home=" storm-home)
+ (str "-Dlogback.configurationFile=" storm-home "/logback/cluster.xml")
+ (str "-Dstorm.id=" storm-id)
+ (str "-Dworker.id=" worker-id)
+ (str "-Dworker.port=" port)
+ "-cp" classpath
+ "backtype.storm.daemon.worker"
+ storm-id
+ (:assignment-id supervisor)
+ port
+ worker-id])
command (->> command (map str) (filter (complement empty?)))
shell-cmd (->> command (map #(str \' % \')) (clojure.string/join " "))]
(log-message "Launching worker with command: " shell-cmd)
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/c970225d/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 281ae52..1fc0d78 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -459,8 +459,7 @@ public class Config extends HashMap<String, Object> {
* with an identifier for this worker.
*/
public static final String WORKER_CHILDOPTS = "worker.childopts";
- public static final Object WORKER_CHILDOPTS_SCHEMA = String.class;
-
+ public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
/**
* How often this worker should heartbeat to the supervisor.
@@ -662,7 +661,7 @@ public class Config extends HashMap<String, Object> {
* Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS.
*/
public static final String TOPOLOGY_WORKER_CHILDOPTS="topology.worker.childopts";
- public static final Object TOPOLOGY_WORKER_CHILDOPTS_SCHEMA = String.class;
+ public static final Object TOPOLOGY_WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
/**
* This config is available for TransactionalSpouts, and contains the id ( a String) for
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/c970225d/storm-core/src/jvm/backtype/storm/ConfigValidation.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/ConfigValidation.java b/storm-core/src/jvm/backtype/storm/ConfigValidation.java
index 15ef6ba..3accb82 100644
--- a/storm-core/src/jvm/backtype/storm/ConfigValidation.java
+++ b/storm-core/src/jvm/backtype/storm/ConfigValidation.java
@@ -135,4 +135,21 @@ public class ConfigValidation {
"Field " + name + " must be an Iterable containing only Strings or Maps of Strings");
}
};
+
+ /**
+ * Validates a String or a list of Strings
+ */
+ public static Object StringOrStringListValidator = new FieldValidator() {
+
+ private FieldValidator fv = FieldListValidatorFactory(String.class);
+
+ @Override
+ public void validateField(String name, Object o) throws IllegalArgumentException {
+ if (o == null || o instanceof String) {
+ // A null value or a String value is acceptable
+ return;
+ }
+ this.fv.validateField(name, o);
+ }
+ };
}
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/c970225d/storm-core/test/clj/backtype/storm/config_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/config_test.clj b/storm-core/test/clj/backtype/storm/config_test.clj
index 93c7df9..01f788b 100644
--- a/storm-core/test/clj/backtype/storm/config_test.clj
+++ b/storm-core/test/clj/backtype/storm/config_test.clj
@@ -83,3 +83,23 @@
(catch Exception e e))))
(is (thrown-cause? java.lang.IllegalArgumentException
(.validateField validator "test" 42)))))
+
+(deftest test-worker-childopts-is-string-or-string-list
+ (let [pass-cases [nil "some string" ["some" "string" "list"]]]
+ (testing "worker.childopts validates"
+ (let [validator (CONFIG-SCHEMA-MAP WORKER-CHILDOPTS)]
+ (doseq [value pass-cases]
+ (is (nil? (try
+ (.validateField validator "test" value)
+ (catch Exception e e)))))
+ (is (thrown-cause? java.lang.IllegalArgumentException
+ (.validateField validator "test" 42)))))
+
+ (testing "topology.worker.childopts validates"
+ (let [validator (CONFIG-SCHEMA-MAP TOPOLOGY-WORKER-CHILDOPTS)]
+ (doseq [value pass-cases]
+ (is (nil? (try
+ (.validateField validator "test" value)
+ (catch Exception e e)))))
+ (is (thrown-cause? java.lang.IllegalArgumentException
+ (.validateField validator "test" 42)))))))
[02/10] git commit: supervisor logs worker command with '-quoted
tokens
Posted by bo...@apache.org.
supervisor logs worker command with '-quoted tokens
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/c7c7087c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/c7c7087c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/c7c7087c
Branch: refs/heads/master
Commit: c7c7087c4be7fff719e846910af1b18c48d01a38
Parents: 193301f
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Sun Jan 12 20:34:51 2014 +0000
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Wed Jan 15 03:16:52 2014 +0000
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/c7c7087c/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 9bff2ad..ea2b460 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -469,8 +469,9 @@
(:assignment-id supervisor)
port
worker-id]
- command (->> command (map str) (filter (complement empty?)))]
- (log-message "Launching worker with command: " (clojure.string/join " " command))
+ command (->> command (map str) (filter (complement empty?)))
+ shell-cmd (->> command (map #(str \' % \')) (clojure.string/join " "))]
+ (log-message "Launching worker with command: " shell-cmd)
(launch-process command :environment {"LD_LIBRARY_PATH" (conf JAVA-LIBRARY-PATH)})
))
[08/10] git commit: escape single quotes when logging
Posted by bo...@apache.org.
escape single quotes when logging
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/061acf17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/061acf17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/061acf17
Branch: refs/heads/master
Commit: 061acf1740d66edd2b90a62635d558a634450c64
Parents: bf557f8
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Tue Apr 15 13:34:13 2014 +0000
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Tue Apr 15 13:34:13 2014 +0000
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/061acf17/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 1630766..0ffb888 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -475,7 +475,9 @@
port
worker-id])
command (->> command (map str) (filter (complement empty?)))
- shell-cmd (->> command (map #(str \' % \')) (clojure.string/join " "))]
+ shell-cmd (->> command
+ (map #(str \' (clojure.string/escape % {\' "\\'"}) \'))
+ (clojure.string/join " "))]
(log-message "Launching worker with command: " shell-cmd)
(launch-process command :environment {"LD_LIBRARY_PATH" (conf JAVA-LIBRARY-PATH)})
))
[04/10] git commit: Do not encode,
decode topo id when launching worker
Posted by bo...@apache.org.
Do not encode,decode topo id when launching worker
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/6d1ad6a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/6d1ad6a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/6d1ad6a6
Branch: refs/heads/master
Commit: 6d1ad6a6a3835294a564ab63b1d6399ee7843e18
Parents: c7c7087
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Sun Jan 12 20:46:46 2014 +0000
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Wed Jan 15 03:16:53 2014 +0000
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 2 +-
storm-core/src/clj/backtype/storm/daemon/worker.clj | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/6d1ad6a6/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index ea2b460..a38ee5c 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -465,7 +465,7 @@
(str "-Dworker.port=" port)
"-cp" classpath
"backtype.storm.daemon.worker"
- (java.net.URLEncoder/encode storm-id)
+ storm-id
(:assignment-id supervisor)
port
worker-id]
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/6d1ad6a6/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/worker.clj b/storm-core/src/clj/backtype/storm/daemon/worker.clj
index 3055b66..0d1f6c6 100644
--- a/storm-core/src/clj/backtype/storm/daemon/worker.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/worker.clj
@@ -451,4 +451,4 @@
(defn -main [storm-id assignment-id port-str worker-id]
(let [conf (read-storm-config)]
(validate-distributed-mode! conf)
- (mk-worker conf nil (java.net.URLDecoder/decode storm-id) assignment-id (Integer/parseInt port-str) worker-id)))
+ (mk-worker conf nil storm-id assignment-id (Integer/parseInt port-str) worker-id)))
[10/10] git commit: Updated Changelog for STORM-194
Posted by bo...@apache.org.
Updated Changelog for STORM-194
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/93612f56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/93612f56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/93612f56
Branch: refs/heads/master
Commit: 93612f5659b3f2ef176f9f46f470a081cdbaf239
Parents: 02b3efd
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Tue Apr 22 16:23:47 2014 +0000
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Tue Apr 22 16:23:47 2014 +0000
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/93612f56/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7ffc085..f3bcd3a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.9.2-incubating (unreleased)
+ * STORM-194: Support list of strings in *.worker.childopts, handle spaces
* STORM-288: Fixes version spelling in pom.xml
* STORM-208: Add storm-kafka as an external module
* STORM-285: Fix storm-core shade plugin config
[09/10] git commit: Merge branch 'd2r-worker-lancher-args' of
https://github.com/d2r/incubator-storm STORM-194: Support list of strings in
*.worker.childopts, handle spaces
Posted by bo...@apache.org.
Merge branch 'd2r-worker-lancher-args' of https://github.com/d2r/incubator-storm
STORM-194: Support list of strings in *.worker.childopts, handle spaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/02b3efd7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/02b3efd7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/02b3efd7
Branch: refs/heads/master
Commit: 02b3efd71c453fd50ee4b95bf490753301ece101
Parents: 8f39495 061acf1
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Tue Apr 22 16:22:58 2014 +0000
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Tue Apr 22 16:22:58 2014 +0000
----------------------------------------------------------------------
pom.xml | 7 +++
storm-core/pom.xml | 5 ++
.../clj/backtype/storm/daemon/supervisor.clj | 46 +++++++++-----
.../src/clj/backtype/storm/daemon/worker.clj | 2 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 3 +-
storm-core/src/clj/backtype/storm/util.clj | 4 +-
storm-core/src/jvm/backtype/storm/Config.java | 5 +-
.../jvm/backtype/storm/ConfigValidation.java | 17 ++++++
.../test/clj/backtype/storm/config_test.clj | 20 ++++++
.../test/clj/backtype/storm/supervisor_test.clj | 64 +++++++++++++++++++-
10 files changed, 149 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/02b3efd7/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/02b3efd7/storm-core/pom.xml
----------------------------------------------------------------------
[06/10] git commit: Add test for handling *.worker.childopts in supvor
Posted by bo...@apache.org.
Add test for handling *.worker.childopts in supvor
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/2fb66fa3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/2fb66fa3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/2fb66fa3
Branch: refs/heads/master
Commit: 2fb66fa3be27f3495fbbe78831565a1edce12af1
Parents: c970225
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Wed Jan 15 03:10:35 2014 +0000
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Wed Jan 15 03:16:54 2014 +0000
----------------------------------------------------------------------
pom.xml | 7 +++
storm-core/pom.xml | 5 ++
.../test/clj/backtype/storm/supervisor_test.clj | 64 +++++++++++++++++++-
3 files changed, 75 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2fb66fa3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3bd3362..3a0c26b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -166,6 +166,7 @@
<clojure-complete.version>0.2.3</clojure-complete.version>
<mockito.version>1.9.5</mockito.version>
<reply.version>0.3.0</reply.version>
+ <conjure.version>2.1.3</conjure.version>
</properties>
@@ -411,6 +412,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.clojars.runa</groupId>
+ <artifactId>conjure</artifactId>
+ <version>${conjure.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.storm</groupId>
<artifactId>libthrift7</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2fb66fa3/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index d333b88..f1a74c6 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -145,6 +145,11 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.clojars.runa</groupId>
+ <artifactId>conjure</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>src/jvm</sourceDirectory>
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2fb66fa3/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index 9f9def9..54a3ee9 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -15,10 +15,12 @@
;; limitations under the License.
(ns backtype.storm.supervisor-test
(:use [clojure test])
+ (:require [clojure [string :as string]])
(:import [backtype.storm.testing TestWordCounter TestWordSpout TestGlobalCount TestAggregatesCounter])
- (:use [backtype.storm bootstrap testing])
+ (:use [backtype.storm bootstrap config testing])
(:use [backtype.storm.daemon common])
(:require [backtype.storm.daemon [worker :as worker] [supervisor :as supervisor]])
+ (:use [conjure core])
)
(bootstrap)
@@ -239,6 +241,66 @@
)))
+(deftest test-worker-launch-command
+ (testing "*.worker.childopts configuration"
+ (let [mock-port "42"
+ mock-storm-id "fake-storm-id"
+ mock-worker-id "fake-worker-id"
+ mock-cp "mock-classpath"
+ exp-args-fn (fn [opts topo-opts]
+ (concat ["java" "-server"]
+ opts
+ topo-opts
+ ["-Djava.library.path="
+ (str "-Dlogfile.name=worker-" mock-port ".log")
+ "-Dstorm.home="
+ "-Dlogback.configurationFile=/logback/cluster.xml"
+ (str "-Dstorm.id=" mock-storm-id)
+ (str "-Dworker.id=" mock-worker-id)
+ (str "-Dworker.port=" mock-port)
+ "-cp" mock-cp
+ "backtype.storm.daemon.worker"
+ mock-storm-id
+ mock-port
+ mock-worker-id]))]
+ (testing "testing *.worker.childopts as strings with extra spaces"
+ (let [string-opts "-Dfoo=bar -Xmx1024m"
+ topo-string-opts "-Dkau=aux -Xmx2048m"
+ exp-args (exp-args-fn ["-Dfoo=bar" "-Xmx1024m"]
+ ["-Dkau=aux" "-Xmx2048m"])
+ mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
+ WORKER-CHILDOPTS string-opts}}]
+ (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
+ topo-string-opts}
+ add-to-classpath mock-cp
+ supervisor-stormdist-root nil
+ launch-process nil]
+ (supervisor/launch-worker mock-supervisor
+ mock-storm-id
+ mock-port
+ mock-worker-id)
+ (verify-first-call-args-for-indices launch-process
+ [0]
+ exp-args))))
+ (testing "testing *.worker.childopts as list of strings, with spaces in values"
+ (let [list-opts '("-Dopt1='this has a space in it'" "-Xmx1024m")
+ topo-list-opts '("-Dopt2='val with spaces'" "-Xmx2048m")
+ exp-args (exp-args-fn list-opts topo-list-opts)
+ mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
+ WORKER-CHILDOPTS list-opts}}]
+ (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
+ topo-list-opts}
+ add-to-classpath mock-cp
+ supervisor-stormdist-root nil
+ launch-process nil]
+ (supervisor/launch-worker mock-supervisor
+ mock-storm-id
+ mock-port
+ mock-worker-id)
+ (verify-first-call-args-for-indices launch-process
+ [0]
+ exp-args)))))))
+
(deftest test-workers-go-bananas
;; test that multiple workers are started for a port, and test that
;; supervisor shuts down propertly (doesn't shutdown the most
[03/10] git commit: URL-decode special chars in topo ids
Posted by bo...@apache.org.
URL-decode special chars in topo ids
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/193301f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/193301f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/193301f2
Branch: refs/heads/master
Commit: 193301f2d4d8188b5cee04f4cf982636ea6c6ba6
Parents: 4738718
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Sun Jan 12 00:50:36 2014 +0000
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Wed Jan 15 03:16:52 2014 +0000
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/ui/core.clj | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/193301f2/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index 63a3053..e46871a 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -29,6 +29,7 @@
Nimbus$Client StormTopology GlobalStreamId RebalanceOptions
KillOptions])
(:import [java.io File])
+ (:import [java.net URLDecoder])
(:require [compojure.route :as route]
[compojure.handler :as handler]
[ring.util.response :as resp]
@@ -1018,32 +1019,33 @@
(GET "/topology/:id" [:as {cookies :cookies} id & m]
(let [include-sys? (get-include-sys? cookies)]
(try
- (-> (topology-page id (:window m) include-sys?)
+ (-> (topology-page (URLDecoder/decode id) (:window m) include-sys?)
(concat [(mk-system-toggle-button include-sys?)])
ui-template)
(catch Exception e (resp/redirect "/")))))
(GET "/topology/:id/component/:component" [:as {cookies :cookies} id component & m]
(let [include-sys? (get-include-sys? cookies)]
- (-> (component-page id component (:window m) include-sys?)
+ (-> (component-page (URLDecoder/decode id)
+ component (:window m) include-sys?)
(concat [(mk-system-toggle-button include-sys?)])
ui-template)))
(POST "/topology/:id/activate" [id]
(with-nimbus nimbus
- (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus (URLDecoder/decode id))
name (.get_name tplg)]
(.activate nimbus name)
(log-message "Activating topology '" name "'")))
(resp/redirect (str "/topology/" id)))
(POST "/topology/:id/deactivate" [id]
(with-nimbus nimbus
- (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus (URLDecoder/decode id))
name (.get_name tplg)]
(.deactivate nimbus name)
(log-message "Deactivating topology '" name "'")))
(resp/redirect (str "/topology/" id)))
(POST "/topology/:id/rebalance/:wait-time" [id wait-time]
(with-nimbus nimbus
- (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus (URLDecoder/decode id))
name (.get_name tplg)
options (RebalanceOptions.)]
(.set_wait_secs options (Integer/parseInt wait-time))
@@ -1052,7 +1054,7 @@
(resp/redirect (str "/topology/" id)))
(POST "/topology/:id/kill/:wait-time" [id wait-time]
(with-nimbus nimbus
- (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus (URLDecoder/decode id))
name (.get_name tplg)
options (KillOptions.)]
(.set_wait_secs options (Integer/parseInt wait-time))