You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by zh...@apache.org on 2015/12/22 00:15:37 UTC
[1/3] storm git commit: [STORM-1401] add test and updates code for
topology history.
Repository: storm
Updated Branches:
refs/heads/master a998c4e9d -> b41f45022
[STORM-1401] add test and updates code for topology history.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/424ef546
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/424ef546
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/424ef546
Branch: refs/heads/master
Commit: 424ef546caa4f564aa5a79a263250062759b75ba
Parents: 418792f
Author: zhuol <zh...@yahoo-inc.com>
Authored: Fri Dec 18 12:56:32 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Fri Dec 18 12:56:32 2015 -0600
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/nimbus.clj | 4 +-
.../test/clj/backtype/storm/nimbus_test.clj | 96 ++++++++++++++++++++
2 files changed, 99 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/424ef546/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index edab3e2..ca24248 100644
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@ -1554,7 +1554,9 @@
(.get_wait_secs options)
)]
(transition-name! nimbus storm-name [:kill wait-amt] true)
- (notify-topology-action-listener nimbus storm-name operation))))
+ (notify-topology-action-listener nimbus storm-name operation))
+ (add-topology-to-history-log (get-storm-id (:storm-cluster-state nimbus) storm-name)
+ nimbus topology-conf)))
(^void rebalance [this ^String storm-name ^RebalanceOptions options]
(mark! nimbus:num-rebalance-calls)
http://git-wip-us.apache.org/repos/asf/storm/blob/424ef546/storm-core/test/clj/backtype/storm/nimbus_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/nimbus_test.clj b/storm-core/test/clj/backtype/storm/nimbus_test.clj
index 4989254..8606902 100644
--- a/storm-core/test/clj/backtype/storm/nimbus_test.clj
+++ b/storm-core/test/clj/backtype/storm/nimbus_test.clj
@@ -397,7 +397,103 @@
(is (= 7 (storm-num-workers state "test")))
)))
+(deftest test-topo-history
+ (with-simulated-time-local-cluster [cluster :supervisors 2 :ports-per-supervisor 5
+ :daemon-conf {SUPERVISOR-ENABLE false
+ NIMBUS-ADMINS ["admin-user"]
+ NIMBUS-TASK-TIMEOUT-SECS 30
+ NIMBUS-MONITOR-FREQ-SECS 10
+ TOPOLOGY-ACKER-EXECUTORS 0}]
+ (stubbing [nimbus/user-groups ["alice-group"]]
+ (letlocals
+ (bind conf (:daemon-conf cluster))
+ (bind topology (thrift/mk-topology
+ {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 4)}
+ {}
+ ))
+ (bind state (:storm-cluster-state cluster))
+ (submit-local-topology (:nimbus cluster) "test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20, LOGS-USERS ["alice", (System/getProperty "user.name")]} topology)
+ (bind storm-id (get-storm-id state "test"))
+ (advance-cluster-time cluster 5)
+ (is (not-nil? (.storm-base state storm-id nil)))
+ (is (not-nil? (.assignment-info state storm-id nil)))
+ (.killTopology (:nimbus cluster) "test")
+ ;; check that storm is deactivated but alive
+ (is (= :killed (-> (.storm-base state storm-id nil) :status :type)))
+ (is (not-nil? (.assignment-info state storm-id nil)))
+ (advance-cluster-time cluster 35)
+ ;; kill topology read on group
+ (submit-local-topology (:nimbus cluster) "killgrouptest" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20, LOGS-GROUPS ["alice-group"]} topology)
+ (bind storm-id-killgroup (get-storm-id state "killgrouptest"))
+ (advance-cluster-time cluster 5)
+ (is (not-nil? (.storm-base state storm-id-killgroup nil)))
+ (is (not-nil? (.assignment-info state storm-id-killgroup nil)))
+ (.killTopology (:nimbus cluster) "killgrouptest")
+ ;; check that storm is deactivated but alive
+ (is (= :killed (-> (.storm-base state storm-id-killgroup nil) :status :type)))
+ (is (not-nil? (.assignment-info state storm-id-killgroup nil)))
+ (advance-cluster-time cluster 35)
+ ;; kill topology can't read
+ (submit-local-topology (:nimbus cluster) "killnoreadtest" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20} topology)
+ (bind storm-id-killnoread (get-storm-id state "killnoreadtest"))
+ (advance-cluster-time cluster 5)
+ (is (not-nil? (.storm-base state storm-id-killnoread nil)))
+ (is (not-nil? (.assignment-info state storm-id-killnoread nil)))
+ (.killTopology (:nimbus cluster) "killnoreadtest")
+ ;; check that storm is deactivated but alive
+ (is (= :killed (-> (.storm-base state storm-id-killnoread nil) :status :type)))
+ (is (not-nil? (.assignment-info state storm-id-killnoread nil)))
+ (advance-cluster-time cluster 35)
+
+ ;; active topology can read
+ (submit-local-topology (:nimbus cluster) "2test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10, LOGS-USERS ["alice", (System/getProperty "user.name")]} topology)
+ (advance-cluster-time cluster 11)
+ (bind storm-id2 (get-storm-id state "2test"))
+ (is (not-nil? (.storm-base state storm-id2 nil)))
+ (is (not-nil? (.assignment-info state storm-id2 nil)))
+ ;; active topology can not read
+ (submit-local-topology (:nimbus cluster) "testnoread" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10, LOGS-USERS ["alice"]} topology)
+ (advance-cluster-time cluster 11)
+ (bind storm-id3 (get-storm-id state "testnoread"))
+ (is (not-nil? (.storm-base state storm-id3 nil)))
+ (is (not-nil? (.assignment-info state storm-id3 nil)))
+ ;; active topology can read based on group
+ (submit-local-topology (:nimbus cluster) "testreadgroup" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10, LOGS-GROUPS ["alice-group"]} topology)
+ (advance-cluster-time cluster 11)
+ (bind storm-id4 (get-storm-id state "testreadgroup"))
+ (is (not-nil? (.storm-base state storm-id4 nil)))
+ (is (not-nil? (.assignment-info state storm-id4 nil)))
+ ;; at this point have 1 running, 1 killed topo
+ (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (:nimbus cluster) (System/getProperty "user.name")))))]
+ (log-message "Checking user " (System/getProperty "user.name") " " hist-topo-ids)
+ (is (= 4 (count hist-topo-ids)))
+ (is (= storm-id2 (get hist-topo-ids 0)))
+ (is (= storm-id-killgroup (get hist-topo-ids 1)))
+ (is (= storm-id (get hist-topo-ids 2)))
+ (is (= storm-id4 (get hist-topo-ids 3))))
+ (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (:nimbus cluster) "alice"))))]
+ (log-message "Checking user alice " hist-topo-ids)
+ (is (= 5 (count hist-topo-ids)))
+ (is (= storm-id2 (get hist-topo-ids 0)))
+ (is (= storm-id-killgroup (get hist-topo-ids 1)))
+ (is (= storm-id (get hist-topo-ids 2)))
+ (is (= storm-id3 (get hist-topo-ids 3)))
+ (is (= storm-id4 (get hist-topo-ids 4))))
+ (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (:nimbus cluster) "admin-user"))))]
+ (log-message "Checking user admin-user " hist-topo-ids)
+ (is (= 6 (count hist-topo-ids)))
+ (is (= storm-id2 (get hist-topo-ids 0)))
+ (is (= storm-id-killgroup (get hist-topo-ids 1)))
+ (is (= storm-id-killnoread (get hist-topo-ids 2)))
+ (is (= storm-id (get hist-topo-ids 3)))
+ (is (= storm-id3 (get hist-topo-ids 4)))
+ (is (= storm-id4 (get hist-topo-ids 5))))
+ (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (:nimbus cluster) "group-only-user"))))]
+ (log-message "Checking user group-only-user " hist-topo-ids)
+ (is (= 2 (count hist-topo-ids)))
+ (is (= storm-id-killgroup (get hist-topo-ids 0)))
+ (is (= storm-id4 (get hist-topo-ids 1))))))))
(deftest test-kill-storm
(with-simulated-time-local-cluster [cluster :supervisors 2 :ports-per-supervisor 5
[2/3] storm git commit: minor
Posted by zh...@apache.org.
minor
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/eb747b35
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/eb747b35
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/eb747b35
Branch: refs/heads/master
Commit: eb747b355643ca2112c63f424738c78c0ef36230
Parents: 424ef54
Author: zhuol <zh...@yahoo-inc.com>
Authored: Fri Dec 18 13:03:17 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Fri Dec 18 13:03:17 2015 -0600
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/nimbus.clj | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/eb747b35/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index ca24248..f25da22 100644
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@ -2159,7 +2159,7 @@
bases (topology-bases storm-cluster-state)
assigned-topology-ids (.assignments storm-cluster-state nil)
user-group-match-fn (fn [topo-id user conf]
- (let [topology-conf (try-read-storm-conf conf topo-id)
+ (let [topology-conf (try-read-storm-conf conf topo-id (:blob-store nimbus))
groups (get-topo-logs-groups topology-conf)]
(or (nil? user)
(some #(= % user) admin-users)
[3/3] storm git commit: Merge branch '1401' of
https://github.com/zhuoliu/storm into STORM-1402-Merge
Posted by zh...@apache.org.
Merge branch '1401' of https://github.com/zhuoliu/storm into STORM-1402-Merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b41f4502
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b41f4502
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b41f4502
Branch: refs/heads/master
Commit: b41f45022e7919093bb431cd2c588dd9245b57c1
Parents: a998c4e eb747b3
Author: zhuol <zh...@yahoo-inc.com>
Authored: Mon Dec 21 17:11:26 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Mon Dec 21 17:11:26 2015 -0600
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/nimbus.clj | 6 +-
.../test/clj/backtype/storm/nimbus_test.clj | 96 ++++++++++++++++++++
2 files changed, 100 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b41f4502/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------