You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/08/25 19:46:45 UTC
[01/14] git commit: Adding substitutes to worker childopts.
Repository: incubator-storm
Updated Branches:
refs/heads/master 14d0ee5c5 -> 2b4cbe91c
Adding substitutes to worker childopts.
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/0db27d5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/0db27d5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/0db27d5b
Branch: refs/heads/master
Commit: 0db27d5b323e8476a989727dbd5cd99444cce56c
Parents: 4c70e40
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Jun 25 14:59:17 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Jun 25 14:59:17 2014 +0000
----------------------------------------------------------------------
.../clj/backtype/storm/daemon/supervisor.clj | 15 +++-
storm-core/src/jvm/backtype/storm/Config.java | 4 +-
.../test/clj/backtype/storm/supervisor_test.clj | 93 ++++++++++++++++++++
3 files changed, 107 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/0db27d5b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 0969970..af88c9d 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -445,8 +445,15 @@
arch-resource-root (str resource-root File/separator os "-" arch)]
(str arch-resource-root File/pathSeparator resource-root File/pathSeparator (conf JAVA-LIBRARY-PATH))))
-(defn- substitute-worker-childopts [value port]
- (let [sub-fn (fn [s] (.replaceAll s "%ID%" (str port)))]
+(defn substitute-childopts [value worker-id storm-id port]
+ (let [replacement-map {"%ID%" (str port)
+ "%WORKER-ID%" (str worker-id)
+ "%STORM-ID%" (str storm-id)
+ "%WORKER-PORT%" (str port)}
+ sub-fn (fn [s]
+ (reduce (fn [string entry]
+ (apply clojure.string/replace string entry))
+ s replacement-map))]
(if (list? value)
(map sub-fn value)
(-> value sub-fn (.split " ")))))
@@ -474,9 +481,9 @@
(add-to-classpath [stormjar])
(add-to-classpath topo-classpath))
worker-childopts (when-let [s (conf WORKER-CHILDOPTS)]
- (substitute-worker-childopts s port))
+ (substitute-childopts s worker-id storm-id port))
topo-worker-childopts (when-let [s (storm-conf TOPOLOGY-WORKER-CHILDOPTS)]
- (substitute-worker-childopts s port))
+ (substitute-childopts s worker-id storm-id port))
topology-worker-environment (if-let [env (storm-conf TOPOLOGY-ENVIRONMENT)]
(merge env {"LD_LIBRARY_PATH" jlp})
{"LD_LIBRARY_PATH" jlp})
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/0db27d5b/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index eefaee7..2b53e27 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -469,7 +469,9 @@ public class Config extends HashMap<String, Object> {
/**
* The jvm opts provided to workers launched by this supervisor. All "%ID%" substrings are replaced
- * with an identifier for this worker.
+ * with an identifier for this worker. Also, "%WORKER-ID%", "%STORM-ID%" and "%WORKER-PORT%" are
+ * replaced with appropriate runtime values for this worker.
+ *
*/
public static final String WORKER_CHILDOPTS = "worker.childopts";
public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/0db27d5b/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index eb69f11..b7cb31e 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -352,3 +352,96 @@
;; TODO just do reassign, and check that cleans up worker states after killing but doesn't get rid of downloaded code
)
+(defn found? [sub-str input-str]
+ (if (string? input-str)
+ (.contains input-str sub-str)
+ (some? #(.substring % sub-str) input-str)))
+
+(defn not-found? [sub-str input-str]
+ (complement (found? sub-str input-str)))
+
+(deftest test-substitute-childopts-happy-path
+ (testing "worker-launcher replaces ids in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
+ ]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
+ )))
+
+(deftest test-substitute-childopts-storm-id-alone
+ (testing "worker-launcher replaces ids in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (not-found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (not-found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids)) )))
+
+(deftest test-substitute-childopts-no-keys
+ (testing "worker-launcher has no ids to replace in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (not-found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (not-found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (not-found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids)) )))
+
+(deftest test-substitute-childopts-nil-childopts
+ (testing "worker-launcher has nil childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts nil]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (not-found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (not-found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (not-found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
+ )))
+
+(deftest test-substitute-childopts-nil-ids
+ (testing "worker-launcher has nil ids"
+ (let [ worker-id nil
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"]
+ (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
+ (is (not-found? "%WORKER-ID%" childopts-with-ids))
+ (is (not-found? "w-01" childopts-with-ids))
+ (is (not-found? "%STORM-ID%" childopts-with-ids))
+ (is (found? "s-01" childopts-with-ids))
+ (is (not-found? "%WORKER-PORT%" childopts-with-ids))
+ (is (found? "-9999." childopts-with-ids))
+ (is (not-found? "%ID%" childopts-with-ids))
+ (is (found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
+ )))
+
+
[02/14] git commit: Correcting function formatting
Posted by bo...@apache.org.
Correcting function formatting
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/cbb0b8a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/cbb0b8a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/cbb0b8a2
Branch: refs/heads/master
Commit: cbb0b8a2970a2094320915934113e4c06a559222
Parents: 0db27d5
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Jun 25 15:18:15 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Jun 25 15:18:15 2014 +0000
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 8 +++++---
storm-core/src/jvm/backtype/storm/Config.java | 1 -
storm-core/test/clj/backtype/storm/supervisor_test.clj | 8 +++++---
3 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/cbb0b8a2/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index af88c9d..620ed5a 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -445,12 +445,14 @@
arch-resource-root (str resource-root File/separator os "-" arch)]
(str arch-resource-root File/pathSeparator resource-root File/pathSeparator (conf JAVA-LIBRARY-PATH))))
-(defn substitute-childopts [value worker-id storm-id port]
- (let [replacement-map {"%ID%" (str port)
+(defn substitute-childopts
+ [value worker-id storm-id port]
+ (let [replacement-map {"%ID%" (str port)
"%WORKER-ID%" (str worker-id)
"%STORM-ID%" (str storm-id)
"%WORKER-PORT%" (str port)}
- sub-fn (fn [s]
+ sub-fn (fn
+ [s]
(reduce (fn [string entry]
(apply clojure.string/replace string entry))
s replacement-map))]
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/cbb0b8a2/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 2b53e27..458502e 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -471,7 +471,6 @@ public class Config extends HashMap<String, Object> {
* The jvm opts provided to workers launched by this supervisor. All "%ID%" substrings are replaced
* with an identifier for this worker. Also, "%WORKER-ID%", "%STORM-ID%" and "%WORKER-PORT%" are
* replaced with appropriate runtime values for this worker.
- *
*/
public static final String WORKER_CHILDOPTS = "worker.childopts";
public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/cbb0b8a2/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index b7cb31e..7c397bb 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -352,13 +352,15 @@
;; TODO just do reassign, and check that cleans up worker states after killing but doesn't get rid of downloaded code
)
-(defn found? [sub-str input-str]
+(defn found?
+ [sub-str input-str]
(if (string? input-str)
(.contains input-str sub-str)
(some? #(.substring % sub-str) input-str)))
-(defn not-found? [sub-str input-str]
- (complement (found? sub-str input-str)))
+(defn not-found?
+ [sub-str input-str]
+ (complement (found? sub-str input-str)))
(deftest test-substitute-childopts-happy-path
(testing "worker-launcher replaces ids in childopts"
[13/14] git commit: Merge branch 'childopts-replacements' of
https://github.com/kishorvpatil/incubator-storm into STORM-373
Posted by bo...@apache.org.
Merge branch 'childopts-replacements' of https://github.com/kishorvpatil/incubator-storm into STORM-373
STORM-373: Provide Additional String substitutions for *.worker.childopts
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/b7fb68c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/b7fb68c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/b7fb68c5
Branch: refs/heads/master
Commit: b7fb68c5cb7866c63af437a8a7d63217517393d0
Parents: 14d0ee5 1d84194
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon Aug 25 11:34:20 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon Aug 25 11:34:20 2014 -0500
----------------------------------------------------------------------
.../clj/backtype/storm/daemon/supervisor.clj | 24 +++++---
storm-core/src/jvm/backtype/storm/Config.java | 8 ++-
.../test/clj/backtype/storm/supervisor_test.clj | 60 ++++++++++++++++++++
3 files changed, 83 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/b7fb68c5/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/b7fb68c5/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/b7fb68c5/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
[14/14] git commit: Added STORM-373 to Changelog
Posted by bo...@apache.org.
Added STORM-373 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/2b4cbe91
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/2b4cbe91
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/2b4cbe91
Branch: refs/heads/master
Commit: 2b4cbe91cd4f42649c06fb75293397c8eb5de669
Parents: b7fb68c
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon Aug 25 11:34:51 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon Aug 25 11:34:51 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2b4cbe91/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7a02dbd..f36ebe8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,6 +56,7 @@
* STORM-376: Add compression to serialization
* STORM-437: Enforce utf-8 when multilang reads from stdin
* STORM-361: Add JSON-P support to Storm UI API
+ * STORM-373: Provide Additional String substitutions for *.worker.childopts
## 0.9.2-incubating
* STORM-66: send taskid on initial handshake
[09/14] git commit: Fix config description for worker.childopts
Posted by bo...@apache.org.
Fix config description for worker.childopts
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/b20fcffc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/b20fcffc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/b20fcffc
Branch: refs/heads/master
Commit: b20fcffcbef3d8764a07e25c08a366cc85ad7afa
Parents: 1c297d7
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Thu Jul 24 21:33:20 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Thu Jul 24 21:33:20 2014 +0000
----------------------------------------------------------------------
storm-core/src/jvm/backtype/storm/Config.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/b20fcffc/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 1afbc7f..57c1a49 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -468,8 +468,7 @@ public class Config extends HashMap<String, Object> {
/**
* The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%STORM-ID%"
- * and "%WORKER-PORT%" substrings are replaced with an identifier for this worker. Each of these ids are
- * replaced with:
+ * and "%WORKER-PORT%" substrings are replaced with:
* %ID% -> port (for backward compatibility),
* %WORKER-ID% -> worker-id,
* %STORM-ID% -> topology-id,
[05/14] git commit: Remove unwanted functions from supervisor-test
Posted by bo...@apache.org.
Remove unwanted functions from supervisor-test
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/b5500ec8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/b5500ec8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/b5500ec8
Branch: refs/heads/master
Commit: b5500ec8f7f0171e6ec37de1e85b2041d2f482af
Parents: c9a6f65
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Tue Jul 22 18:51:34 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Tue Jul 22 18:51:34 2014 +0000
----------------------------------------------------------------------
.../test/clj/backtype/storm/supervisor_test.clj | 30 +++++++-------------
1 file changed, 10 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/b5500ec8/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index f597284..ea08c86 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -352,25 +352,15 @@
;; TODO just do reassign, and check that cleans up worker states after killing but doesn't get rid of downloaded code
)
-(defn found?
- [sub-str input-str]
- (if (string? input-str)
- (.contains input-str sub-str)
- (some? #(.substring % sub-str) input-str)))
-
-(defn not-found?
- [sub-str input-str]
- (complement (found? sub-str input-str)))
-
(deftest test-substitute-childopts-happy-path-string
(testing "worker-launcher replaces ids in childopts"
(let [ worker-id "w-01"
storm-id "s-01"
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m"]
childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (into [] childopts-with-ids))))))
+ (is (= expected-childopts (vec childopts-with-ids))))))
(deftest test-substitute-childopts-happy-path-list
(testing "worker-launcher replaces ids in childopts"
@@ -378,9 +368,9 @@
storm-id "s-01"
port 9999
childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m"]
childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (into [] childopts-with-ids))))))
+ (is (= expected-childopts (vec childopts-with-ids))))))
(deftest test-substitute-childopts-storm-id-alone
(testing "worker-launcher replaces ids in childopts"
@@ -388,9 +378,9 @@
storm-id "s-01"
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
+ expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log"]
childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (into [] childopts-with-ids))))))
+ (is (= expected-childopts (vec childopts-with-ids))))))
(deftest test-substitute-childopts-no-keys
(testing "worker-launcher has no ids to replace in childopts"
@@ -398,9 +388,9 @@
storm-id "s-01"
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
+ expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"]
childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (into [] childopts-with-ids))))))
+ (is (= expected-childopts (vec childopts-with-ids))))))
(deftest test-substitute-childopts-nil-childopts
(testing "worker-launcher has nil childopts"
@@ -418,7 +408,7 @@
storm-id "s-01"
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
+ expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log"]
childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (into [] childopts-with-ids))))))
+ (is (= expected-childopts (vec childopts-with-ids))))))
[12/14] git commit: Use TOPOLOGY-ID instead of STORM-ID
Posted by bo...@apache.org.
Use TOPOLOGY-ID instead of STORM-ID
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/1d841941
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/1d841941
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/1d841941
Branch: refs/heads/master
Commit: 1d841941a848bfd17e9a830413415cca21ff76f8
Parents: 32c671a
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Aug 20 21:14:52 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Aug 20 21:14:52 2014 +0000
----------------------------------------------------------------------
.../clj/backtype/storm/daemon/supervisor.clj | 6 ++--
storm-core/src/jvm/backtype/storm/Config.java | 4 +--
.../test/clj/backtype/storm/supervisor_test.clj | 34 ++++++++++----------
3 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/1d841941/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 362452e..4ba9811 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -474,11 +474,11 @@
(str arch-resource-root File/pathSeparator resource-root File/pathSeparator (conf JAVA-LIBRARY-PATH))))
(defn substitute-childopts
- "Generates runtime childopts by replacing keys with storm-id, worker-id, port"
- [value worker-id storm-id port]
+ "Generates runtime childopts by replacing keys with topology-id, worker-id, port"
+ [value worker-id topology-id port]
(let [replacement-map {"%ID%" (str port)
"%WORKER-ID%" (str worker-id)
- "%STORM-ID%" (str storm-id)
+ "%TOPOLOGY-ID%" (str topology-id)
"%WORKER-PORT%" (str port)}
sub-fn #(reduce (fn [string entry]
(apply clojure.string/replace string entry))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/1d841941/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 57c1a49..5d7f762 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -467,11 +467,11 @@ public class Config extends HashMap<String, Object> {
public static final Object SUPERVISOR_MONITOR_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%STORM-ID%"
+ * The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%TOPOLOGY-ID%"
* and "%WORKER-PORT%" substrings are replaced with:
* %ID% -> port (for backward compatibility),
* %WORKER-ID% -> worker-id,
- * %STORM-ID% -> topology-id,
+ * %TOPOLOGY-ID% -> topology-id,
* %WORKER-PORT% -> port.
*/
public static final String WORKER_CHILDOPTS = "worker.childopts";
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/1d841941/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index 68ac09c..1c19aed 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -356,61 +356,61 @@
(deftest test-substitute-childopts-happy-path-string
(testing "worker-launcher replaces ids in childopts"
(let [worker-id "w-01"
- storm-id "s-01"
+ topology-id "s-01"
port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-happy-path-list
(testing "worker-launcher replaces ids in childopts"
(let [worker-id "w-01"
- storm-id "s-01"
+ topology-id "s-01"
port 9999
- childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
+ childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port)]
(is (= expected-childopts childopts-with-ids)))))
-(deftest test-substitute-childopts-storm-id-alone
+(deftest test-substitute-childopts-topology-id-alone
(testing "worker-launcher replaces ids in childopts"
(let [worker-id "w-01"
- storm-id "s-01"
+ topology-id "s-01"
port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%TOPOLOGY-ID%.log"
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-no-keys
(testing "worker-launcher has no ids to replace in childopts"
(let [worker-id "w-01"
- storm-id "s-01"
+ topology-id "s-01"
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-nil-childopts
(testing "worker-launcher has nil childopts"
(let [worker-id "w-01"
- storm-id "s-01"
+ topology-id "s-01"
port 9999
childopts nil
expected-childopts nil
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-nil-ids
(testing "worker-launcher has nil ids"
(let [worker-id nil
- storm-id "s-01"
+ topology-id "s-01"
port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log"
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-retry-read-assignments
[07/14] git commit: Refactor substitute-childopts to use
clojure.string/split and fix unit tests
Posted by bo...@apache.org.
Refactor substitute-childopts to use clojure.string/split and fix unit tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/a0232c85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/a0232c85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/a0232c85
Branch: refs/heads/master
Commit: a0232c85940529860d5ccaa5db82ee95927116fb
Parents: 32a49a9
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Tue Jul 22 20:56:44 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Tue Jul 22 20:56:44 2014 +0000
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 2 +-
storm-core/test/clj/backtype/storm/supervisor_test.clj | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/a0232c85/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 8f78091..ecffcee 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -476,7 +476,7 @@
(cond
(nil? value) nil
(list? value) (map sub-fn value)
- :else (-> value sub-fn (.split " ")))))
+ :else (-> value sub-fn (clojure.string/split #"\s+")))))
(defn java-cmd []
(let [java-home (.get (System/getenv) "JAVA_HOME")]
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/a0232c85/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index a22c909..2607d57 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -359,7 +359,7 @@
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
- childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-happy-path-list
@@ -369,7 +369,7 @@
port 9999
childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
- childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-storm-id-alone
@@ -379,7 +379,7 @@
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
- childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-no-keys
@@ -389,7 +389,7 @@
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
- childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-nil-childopts
@@ -409,6 +409,6 @@
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
- childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
[08/14] git commit: Reformatting function and comments
Posted by bo...@apache.org.
Reformatting function and comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/1c297d70
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/1c297d70
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/1c297d70
Branch: refs/heads/master
Commit: 1c297d70993014a4713f0fa6584a725c480e1cfd
Parents: a0232c8
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Tue Jul 22 21:57:09 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Tue Jul 22 21:57:35 2014 +0000
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/supervisor.clj | 18 +++++++++---------
storm-core/src/jvm/backtype/storm/Config.java | 8 ++++----
2 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/1c297d70/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index ecffcee..5336508 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -465,18 +465,18 @@
(defn substitute-childopts
"Generates runtime childopts by replacing keys with storm-id, worker-id, port"
[value worker-id storm-id port]
- (let [replacement-map {"%ID%" (str port)
- "%WORKER-ID%" (str worker-id)
- "%STORM-ID%" (str storm-id)
- "%WORKER-PORT%" (str port)}
- sub-fn #(reduce (fn [string entry]
- (apply clojure.string/replace string entry))
- %
- replacement-map)]
+ (let [replacement-map {"%ID%" (str port)
+ "%WORKER-ID%" (str worker-id)
+ "%STORM-ID%" (str storm-id)
+ "%WORKER-PORT%" (str port)}
+ sub-fn #(reduce (fn [string entry]
+ (apply clojure.string/replace string entry))
+ %
+ replacement-map)]
(cond
(nil? value) nil
(list? value) (map sub-fn value)
- :else (-> value sub-fn (clojure.string/split #"\s+")))))
+ :else (-> value sub-fn (clojure.string/split #"\s+")))))
(defn java-cmd []
(let [java-home (.get (System/getenv) "JAVA_HOME")]
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/1c297d70/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 06a1f9b..1afbc7f 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -470,10 +470,10 @@ public class Config extends HashMap<String, Object> {
* The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%STORM-ID%"
* and "%WORKER-PORT%" substrings are replaced with an identifier for this worker. Each of these ids are
* replaced with:
- * %ID% -> port ( for backword compatibilty).
- * "%WORKER-ID%" -> worker-id,
- * "%STORM-ID%" -> topology-id,
- * "%WORKER-PORT%" -> port.
+ * %ID% -> port (for backward compatibility),
+ * %WORKER-ID% -> worker-id,
+ * %STORM-ID% -> topology-id,
+ * %WORKER-PORT% -> port.
*/
public static final String WORKER_CHILDOPTS = "worker.childopts";
public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
[06/14] git commit: Refactor method substitute-childopts to use cond
and cleaner asserts in unit tests
Posted by bo...@apache.org.
Refactor method substitute-childopts to use cond and cleaner asserts in unit tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/32a49a96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/32a49a96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/32a49a96
Branch: refs/heads/master
Commit: 32a49a965c314c4de4ccec0ecb4673177296fdad
Parents: b5500ec
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Tue Jul 22 20:37:28 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Tue Jul 22 20:37:28 2014 +0000
----------------------------------------------------------------------
.../clj/backtype/storm/daemon/supervisor.clj | 9 +++---
.../test/clj/backtype/storm/supervisor_test.clj | 30 ++++++++++----------
2 files changed, 19 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/32a49a96/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index db7a284..8f78091 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -473,11 +473,10 @@
(apply clojure.string/replace string entry))
%
replacement-map)]
- (if-not (nil? value)
- (if (list? value)
- (map sub-fn value)
- (-> value sub-fn (.split " ")))
- nil)))
+ (cond
+ (nil? value) nil
+ (list? value) (map sub-fn value)
+ :else (-> value sub-fn (.split " ")))))
(defn java-cmd []
(let [java-home (.get (System/getenv) "JAVA_HOME")]
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/32a49a96/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index ea08c86..a22c909 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -358,9 +358,9 @@
storm-id "s-01"
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
- expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m"]
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (vec childopts-with-ids))))))
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ (is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-happy-path-list
(testing "worker-launcher replaces ids in childopts"
@@ -368,9 +368,9 @@
storm-id "s-01"
port 9999
childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
- expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m"]
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (vec childopts-with-ids))))))
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ (is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-storm-id-alone
(testing "worker-launcher replaces ids in childopts"
@@ -378,9 +378,9 @@
storm-id "s-01"
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"
- expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log"]
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (vec childopts-with-ids))))))
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
+ childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ (is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-no-keys
(testing "worker-launcher has no ids to replace in childopts"
@@ -388,9 +388,9 @@
storm-id "s-01"
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
- expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"]
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (vec childopts-with-ids))))))
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
+ childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ (is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-nil-childopts
(testing "worker-launcher has nil childopts"
@@ -408,7 +408,7 @@
storm-id "s-01"
port 9999
childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
- expected-childopts ["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log"]
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
- (is (= expected-childopts (vec childopts-with-ids))))))
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
+ childopts-with-ids (seq (supervisor/substitute-childopts childopts worker-id storm-id port))]
+ (is (= expected-childopts childopts-with-ids)))))
[03/14] git commit: Merge branch 'master' into childopts-replacements
Posted by bo...@apache.org.
Merge branch 'master' into childopts-replacements
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/20f4f5ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/20f4f5ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/20f4f5ea
Branch: refs/heads/master
Commit: 20f4f5eadf832c1d3f5d6cff9e7243c7a844d3d0
Parents: cbb0b8a 85c9cd5
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Fri Jul 18 21:16:53 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Fri Jul 18 21:16:53 2014 +0000
----------------------------------------------------------------------
BYLAWS.md | 96 ++++
CHANGELOG.md | 17 +
LICENSE | 2 +-
README.markdown | 7 +
STORM-UI-REST-API.md | 546 +++++++++++++++++++
bin/storm | 77 ++-
conf/storm_env.ini | 2 +-
.../storm-kafka/src/jvm/storm/kafka/Broker.java | 9 +-
.../src/jvm/storm/kafka/Partition.java | 9 +-
pom.xml | 6 +-
.../src/clj/backtype/storm/LocalCluster.clj | 7 +-
storm-core/src/clj/backtype/storm/cluster.clj | 31 +-
.../src/clj/backtype/storm/command/monitor.clj | 37 ++
.../src/clj/backtype/storm/daemon/common.clj | 2 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 7 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 181 ++++--
.../src/clj/backtype/storm/daemon/nimbus.clj | 6 +-
.../clj/backtype/storm/daemon/supervisor.clj | 42 +-
.../src/clj/backtype/storm/daemon/worker.clj | 27 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 2 +-
storm-core/src/clj/backtype/storm/event.clj | 2 +-
storm-core/src/clj/backtype/storm/testing.clj | 77 +--
.../src/clj/backtype/storm/ui/helpers.clj | 5 +
storm-core/src/clj/backtype/storm/util.clj | 50 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 25 +
.../src/dev/resources/tester_bolt_metrics.py | 35 ++
.../src/dev/resources/tester_spout_metrics.py | 51 ++
storm-core/src/jvm/backtype/storm/Config.java | 121 ++--
.../jvm/backtype/storm/ConfigValidation.java | 70 +++
.../metric/api/rpc/AssignableShellMetric.java | 30 +
.../metric/api/rpc/CombinedShellMetric.java | 31 ++
.../storm/metric/api/rpc/CountShellMetric.java | 38 ++
.../storm/metric/api/rpc/IShellMetric.java | 31 ++
.../metric/api/rpc/ReducedShellMetric.java | 32 ++
.../storm/multilang/JsonSerializer.java | 10 +
.../jvm/backtype/storm/multilang/ShellMsg.java | 20 +
.../jvm/backtype/storm/spout/ShellSpout.java | 44 +-
.../src/jvm/backtype/storm/task/ShellBolt.java | 47 +-
.../backtype/storm/task/TopologyContext.java | 24 +
.../storm/testing/PythonShellMetricsBolt.java | 32 ++
.../storm/testing/PythonShellMetricsSpout.java | 35 ++
.../storm/ui/InvalidRequestException.java | 20 +
.../src/jvm/backtype/storm/utils/Monitor.java | 249 +++++++++
.../jvm/backtype/storm/utils/ShellProcess.java | 46 +-
.../src/jvm/backtype/storm/utils/Utils.java | 23 +-
storm-core/src/multilang/py/storm.py | 11 +-
storm-core/src/ui/public/component.html | 22 +-
storm-core/src/ui/public/css/style.css | 1 +
.../src/ui/public/js/jquery.tablesorter.min.js | 9 +-
storm-core/src/ui/public/topology.html | 4 +-
.../test/clj/backtype/storm/config_test.clj | 41 +-
.../test/clj/backtype/storm/metrics_test.clj | 206 ++++---
52 files changed, 2235 insertions(+), 320 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/20f4f5ea/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/20f4f5ea/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
[04/14] git commit: Simplify childopts function and clean-up unit
tests
Posted by bo...@apache.org.
Simplify childopts function and clean-up unit tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/c9a6f650
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/c9a6f650
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/c9a6f650
Branch: refs/heads/master
Commit: c9a6f650c23f39dd3303a19a06edc45d9e97a4d4
Parents: 20f4f5e
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Tue Jul 22 16:27:27 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Tue Jul 22 16:27:27 2014 +0000
----------------------------------------------------------------------
.../clj/backtype/storm/daemon/supervisor.clj | 18 ++--
storm-core/src/jvm/backtype/storm/Config.java | 10 ++-
.../test/clj/backtype/storm/supervisor_test.clj | 87 +++++++-------------
3 files changed, 48 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/c9a6f650/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index f467903..db7a284 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -463,19 +463,21 @@
(str arch-resource-root File/pathSeparator resource-root File/pathSeparator (conf JAVA-LIBRARY-PATH))))
(defn substitute-childopts
+ "Generates runtime childopts by replacing keys with storm-id, worker-id, port"
[value worker-id storm-id port]
(let [replacement-map {"%ID%" (str port)
"%WORKER-ID%" (str worker-id)
"%STORM-ID%" (str storm-id)
"%WORKER-PORT%" (str port)}
- sub-fn (fn
- [s]
- (reduce (fn [string entry]
- (apply clojure.string/replace string entry))
- s replacement-map))]
- (if (list? value)
- (map sub-fn value)
- (-> value sub-fn (.split " ")))))
+ sub-fn #(reduce (fn [string entry]
+ (apply clojure.string/replace string entry))
+ %
+ replacement-map)]
+ (if-not (nil? value)
+ (if (list? value)
+ (map sub-fn value)
+ (-> value sub-fn (.split " ")))
+ nil)))
(defn java-cmd []
(let [java-home (.get (System/getenv) "JAVA_HOME")]
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/c9a6f650/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 67a438a..06a1f9b 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -467,9 +467,13 @@ public class Config extends HashMap<String, Object> {
public static final Object SUPERVISOR_MONITOR_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
/**
- * The jvm opts provided to workers launched by this supervisor. All "%ID%" substrings are replaced
- * with an identifier for this worker. Also, "%WORKER-ID%", "%STORM-ID%" and "%WORKER-PORT%" are
- * replaced with appropriate runtime values for this worker.
+ * The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%STORM-ID%"
+ * and "%WORKER-PORT%" substrings are replaced with an identifier for this worker. Each of these ids are
+ * replaced with:
+ * %ID% -> port ( for backword compatibilty).
+ * "%WORKER-ID%" -> worker-id,
+ * "%STORM-ID%" -> topology-id,
+ * "%WORKER-PORT%" -> port.
*/
public static final String WORKER_CHILDOPTS = "worker.childopts";
public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/c9a6f650/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index 7c397bb..f597284 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -362,88 +362,63 @@
[sub-str input-str]
(complement (found? sub-str input-str)))
-(deftest test-substitute-childopts-happy-path
+(deftest test-substitute-childopts-happy-path-string
(testing "worker-launcher replaces ids in childopts"
(let [ worker-id "w-01"
storm-id "s-01"
port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
- ]
- (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
- (is (not-found? "%WORKER-ID%" childopts-with-ids))
- (is (found? "w-01" childopts-with-ids))
- (is (not-found? "%STORM-ID%" childopts-with-ids))
- (is (found? "s-01" childopts-with-ids))
- (is (not-found? "%WORKER-PORT%" childopts-with-ids))
- (is (found? "-9999." childopts-with-ids))
- (is (not-found? "%ID%" childopts-with-ids))
- (is (found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
- )))
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts (into [] childopts-with-ids))))))
+
+(deftest test-substitute-childopts-happy-path-list
+ (testing "worker-launcher replaces ids in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts (into [] childopts-with-ids))))))
(deftest test-substitute-childopts-storm-id-alone
(testing "worker-launcher replaces ids in childopts"
(let [ worker-id "w-01"
storm-id "s-01"
port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"]
- (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
- (is (not-found? "%WORKER-ID%" childopts-with-ids))
- (is (not-found? "w-01" childopts-with-ids))
- (is (not-found? "%STORM-ID%" childopts-with-ids))
- (is (found? "s-01" childopts-with-ids))
- (is (not-found? "%WORKER-PORT%" childopts-with-ids))
- (is (not-found? "-9999." childopts-with-ids))
- (is (not-found? "%ID%" childopts-with-ids))
- (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids)) )))
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts (into [] childopts-with-ids))))))
(deftest test-substitute-childopts-no-keys
(testing "worker-launcher has no ids to replace in childopts"
(let [ worker-id "w-01"
storm-id "s-01"
port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"]
- (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
- (is (not-found? "%WORKER-ID%" childopts-with-ids))
- (is (not-found? "w-01" childopts-with-ids))
- (is (not-found? "%STORM-ID%" childopts-with-ids))
- (is (not-found? "s-01" childopts-with-ids))
- (is (not-found? "%WORKER-PORT%" childopts-with-ids))
- (is (not-found? "-9999." childopts-with-ids))
- (is (not-found? "%ID%" childopts-with-ids))
- (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids)) )))
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts (into [] childopts-with-ids))))))
(deftest test-substitute-childopts-nil-childopts
(testing "worker-launcher has nil childopts"
(let [ worker-id "w-01"
storm-id "s-01"
port 9999
- childopts nil]
- (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
- (is (not-found? "%WORKER-ID%" childopts-with-ids))
- (is (not-found? "w-01" childopts-with-ids))
- (is (not-found? "%STORM-ID%" childopts-with-ids))
- (is (not-found? "s-01" childopts-with-ids))
- (is (not-found? "%WORKER-PORT%" childopts-with-ids))
- (is (not-found? "-9999." childopts-with-ids))
- (is (not-found? "%ID%" childopts-with-ids))
- (is (not-found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
- )))
+ childopts nil
+ expected-childopts nil
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-nil-ids
(testing "worker-launcher has nil ids"
(let [ worker-id nil
storm-id "s-01"
port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"]
- (def childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port))
- (is (not-found? "%WORKER-ID%" childopts-with-ids))
- (is (not-found? "w-01" childopts-with-ids))
- (is (not-found? "%STORM-ID%" childopts-with-ids))
- (is (found? "s-01" childopts-with-ids))
- (is (not-found? "%WORKER-PORT%" childopts-with-ids))
- (is (found? "-9999." childopts-with-ids))
- (is (not-found? "%ID%" childopts-with-ids))
- (is (found? "worker-9999" childopts-with-ids) (str childopts-with-ids))
- )))
-
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts (into [] childopts-with-ids))))))
[11/14] git commit: Remove unwanted spaces
Posted by bo...@apache.org.
Remove unwanted spaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/32c671ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/32c671ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/32c671ad
Branch: refs/heads/master
Commit: 32c671ad173b4e33b1be1ef1a0e279808b06e3b8
Parents: b2aa030
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Aug 20 20:00:03 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Aug 20 20:00:03 2014 +0000
----------------------------------------------------------------------
.../test/clj/backtype/storm/supervisor_test.clj | 72 ++++++++++----------
1 file changed, 36 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/32c671ad/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index 5f65ae7..68ac09c 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -355,62 +355,62 @@
(deftest test-substitute-childopts-happy-path-string
(testing "worker-launcher replaces ids in childopts"
- (let [ worker-id "w-01"
- storm-id "s-01"
- port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (let [worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-happy-path-list
(testing "worker-launcher replaces ids in childopts"
- (let [ worker-id "w-01"
- storm-id "s-01"
- port 9999
- childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (let [worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-storm-id-alone
(testing "worker-launcher replaces ids in childopts"
- (let [ worker-id "w-01"
- storm-id "s-01"
- port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (let [worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-no-keys
(testing "worker-launcher has no ids to replace in childopts"
- (let [ worker-id "w-01"
- storm-id "s-01"
- port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (let [worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-nil-childopts
(testing "worker-launcher has nil childopts"
- (let [ worker-id "w-01"
- storm-id "s-01"
- port 9999
- childopts nil
- expected-childopts nil
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (let [worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts nil
+ expected-childopts nil
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-substitute-childopts-nil-ids
(testing "worker-launcher has nil ids"
- (let [ worker-id nil
- storm-id "s-01"
- port 9999
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (let [worker-id nil
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
(is (= expected-childopts childopts-with-ids)))))
(deftest test-retry-read-assignments
[10/14] git commit: Merge branch 'master' into childopts-replacements
Posted by bo...@apache.org.
Merge branch 'master' into childopts-replacements
Conflicts:
storm-core/test/clj/backtype/storm/supervisor_test.clj
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/b2aa030c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/b2aa030c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/b2aa030c
Branch: refs/heads/master
Commit: b2aa030c478875411be009119b0ebc563a4036a9
Parents: b20fcff b2a8a77
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Jul 30 21:40:59 2014 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Jul 30 21:40:59 2014 +0000
----------------------------------------------------------------------
CHANGELOG.md | 13 ++
LICENSE | 28 +++
README.markdown | 7 +
STORM-UI-REST-API.md | 23 +-
conf/defaults.yaml | 3 +-
dev-tools/github/__init__.py | 109 +++++++++
dev-tools/jira-github-join.py | 80 +++++++
dev-tools/jira/__init__.py | 232 +++++++++++++++++++
examples/storm-starter/README.markdown | 3 +
.../storm-starter/multilang/resources/storm.py | 2 +-
.../src/jvm/storm/kafka/KafkaConfig.java | 2 +-
.../src/clj/backtype/storm/daemon/executor.clj | 4 +-
.../clj/backtype/storm/daemon/supervisor.clj | 36 ++-
storm-core/src/clj/backtype/storm/ui/core.clj | 2 +-
.../src/clj/backtype/storm/ui/helpers.clj | 5 -
storm-core/src/clj/backtype/storm/util.clj | 2 +
.../backtype/storm/messaging/netty/Client.java | 13 +-
.../storm/multilang/JsonSerializer.java | 7 +-
.../jvm/backtype/storm/multilang/ShellMsg.java | 26 +++
.../src/jvm/backtype/storm/spout/ISpout.java | 2 +-
.../jvm/backtype/storm/spout/ShellSpout.java | 30 ++-
.../src/jvm/backtype/storm/task/ShellBolt.java | 32 ++-
.../backtype/storm/task/TopologyContext.java | 4 +
.../jvm/backtype/storm/utils/ShellProcess.java | 2 +-
storm-core/src/multilang/py/storm.py | 19 +-
storm-core/src/multilang/rb/storm.rb | 24 +-
storm-core/src/ui/public/component.html | 3 +-
storm-core/src/ui/public/js/moment.min.js | 6 +
storm-core/src/ui/public/js/script.js | 9 +
.../test/clj/backtype/storm/cluster_test.clj | 3 +-
.../test/clj/backtype/storm/supervisor_test.clj | 85 +++++--
31 files changed, 761 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/b2aa030c/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/b2aa030c/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index 2607d57,584f0d9..5f65ae7
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -352,63 -353,59 +353,119 @@@
;; TODO just do reassign, and check that cleans up worker states after killing but doesn't get rid of downloaded code
)
+(deftest test-substitute-childopts-happy-path-string
+ (testing "worker-launcher replaces ids in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
+
+(deftest test-substitute-childopts-happy-path-list
+ (testing "worker-launcher replaces ids in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m")
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
+
+(deftest test-substitute-childopts-storm-id-alone
+ (testing "worker-launcher replaces ids in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%STORM-ID%.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
+
+(deftest test-substitute-childopts-no-keys
+ (testing "worker-launcher has no ids to replace in childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
+
+(deftest test-substitute-childopts-nil-childopts
+ (testing "worker-launcher has nil childopts"
+ (let [ worker-id "w-01"
+ storm-id "s-01"
+ port 9999
+ childopts nil
+ expected-childopts nil
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
+
+(deftest test-substitute-childopts-nil-ids
+ (testing "worker-launcher has nil ids"
+ (let [ worker-id nil
+ storm-id "s-01"
+ port 9999
+ childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%STORM-ID%-%WORKER-ID%-%WORKER-PORT%.log"
+ expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
+ childopts-with-ids (supervisor/substitute-childopts childopts worker-id storm-id port)]
+ (is (= expected-childopts childopts-with-ids)))))
+
+ (deftest test-retry-read-assignments
+ (with-simulated-time-local-cluster [cluster
+ :supervisors 0
+ :ports-per-supervisor 2
+ :daemon-conf {NIMBUS-REASSIGN false
+ NIMBUS-MONITOR-FREQ-SECS 10
+ TOPOLOGY-MESSAGE-TIMEOUT-SECS 30
+ TOPOLOGY-ACKER-EXECUTORS 0}]
+ (letlocals
+ (bind sup1 (add-supervisor cluster :id "sup1" :ports [1 2 3 4]))
+ (bind topology1 (thrift/mk-topology
+ {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 2)}
+ {}))
+ (bind topology2 (thrift/mk-topology
+ {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 2)}
+ {}))
+ (bind state (:storm-cluster-state cluster))
+ (bind changed (capture-changed-workers
+ (submit-mocked-assignment
+ (:nimbus cluster)
+ "topology1"
+ {TOPOLOGY-WORKERS 2}
+ topology1
+ {1 "1"
+ 2 "1"}
+ {[1] ["sup1" 1]
+ [2] ["sup1" 2]
+ })
+ (submit-mocked-assignment
+ (:nimbus cluster)
+ "topology2"
+ {TOPOLOGY-WORKERS 2}
+ topology2
+ {1 "1"
+ 2 "1"}
+ {[1] ["sup1" 1]
+ [2] ["sup1" 2]
+ })
+ (advance-cluster-time cluster 10)
+ ))
+ (is (empty? (:launched changed)))
+ (bind options (RebalanceOptions.))
+ (.set_wait_secs options 0)
+ (bind changed (capture-changed-workers
+ (.rebalance (:nimbus cluster) "topology2" options)
+ (advance-cluster-time cluster 10)
+ (heartbeat-workers cluster "sup1" [1 2 3 4])
+ (advance-cluster-time cluster 10)
+ ))
+ (validate-launched-once (:launched changed)
+ {"sup1" [1 2]}
+ (get-storm-id (:storm-cluster-state cluster) "topology1"))
+ (validate-launched-once (:launched changed)
+ {"sup1" [3 4]}
+ (get-storm-id (:storm-cluster-state cluster) "topology2"))
+ )))