You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/06/12 21:55:00 UTC
[03/13] move towards idiomatic Clojure style
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2278fc96/storm-core/src/clj/backtype/storm/testing.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/testing.clj b/storm-core/src/clj/backtype/storm/testing.clj
index 7bbe238..32f7f88 100644
--- a/storm-core/src/clj/backtype/storm/testing.clj
+++ b/storm-core/src/clj/backtype/storm/testing.clj
@@ -13,6 +13,7 @@
;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;; See the License for the specific language governing permissions and
;; limitations under the License.
+
(ns backtype.storm.testing
(:require [backtype.storm.daemon
[nimbus :as nimbus]
@@ -43,45 +44,48 @@
(:require [backtype.storm.daemon.acker :as acker])
(:use [backtype.storm cluster util thrift config log]))
-(defn feeder-spout [fields]
+(defn feeder-spout
+ [fields]
(FeederSpout. (Fields. fields)))
-(defn local-temp-path []
+(defn local-temp-path
+ []
(str (System/getProperty "java.io.tmpdir") (if-not on-windows? "/") (uuid)))
-(defn delete-all [paths]
+(defn delete-all
+ [paths]
(dorun
(for [t paths]
(if (.exists (File. t))
(try
(FileUtils/forceDelete (File. t))
(catch Exception e
- (log-message (.getMessage e))))
- ))))
+ (log-message (.getMessage e))))))))
-(defmacro with-local-tmp [[& tmp-syms] & body]
+(defmacro with-local-tmp
+ [[& tmp-syms] & body]
(let [tmp-paths (mapcat (fn [t] [t `(local-temp-path)]) tmp-syms)]
`(let [~@tmp-paths]
- (try
- ~@body
- (finally
- (delete-all ~(vec tmp-syms)))
- ))
- ))
-
-(defn start-simulating-time! []
+ (try
+ ~@body
+ (finally
+ (delete-all ~(vec tmp-syms)))))))
+
+(defn start-simulating-time!
+ []
(Time/startSimulating))
-(defn stop-simulating-time! []
+(defn stop-simulating-time!
+ []
(Time/stopSimulating))
-(defmacro with-simulated-time [& body]
+(defmacro with-simulated-time
+ [& body]
`(do
(start-simulating-time!)
(let [ret# (do ~@body)]
(stop-simulating-time!)
- ret#
- )))
+ ret#)))
(defn advance-time-ms! [ms]
(Time/advanceTime ms))
@@ -89,26 +93,25 @@
(defn advance-time-secs! [secs]
(advance-time-ms! (* (long secs) 1000)))
-
-(defnk add-supervisor [cluster-map :ports 2 :conf {} :id nil]
+(defnk add-supervisor
+ [cluster-map :ports 2 :conf {} :id nil]
(let [tmp-dir (local-temp-path)
- port-ids (if (sequential? ports) ports (doall (repeatedly ports (:port-counter cluster-map))))
+ port-ids (if (sequential? ports)
+ ports
+ (doall (repeatedly ports (:port-counter cluster-map))))
supervisor-conf (merge (:daemon-conf cluster-map)
conf
{STORM-LOCAL-DIR tmp-dir
- SUPERVISOR-SLOTS-PORTS port-ids
- })
+ SUPERVISOR-SLOTS-PORTS port-ids})
id-fn (if id (fn [] id) supervisor/generate-supervisor-id)
daemon (with-var-roots [supervisor/generate-supervisor-id id-fn] (supervisor/mk-supervisor supervisor-conf (:shared-context cluster-map) (supervisor/standalone-supervisor)))]
(swap! (:supervisors cluster-map) conj daemon)
(swap! (:tmp-dirs cluster-map) conj tmp-dir)
- daemon
- ))
+ daemon))
(defn mk-shared-context [conf]
(if-not (conf STORM-LOCAL-MODE-ZMQ)
- (msg-loader/mk-local-context)
- ))
+ (msg-loader/mk-local-context)))
;; returns map containing cluster info
;; local dir is always overridden in maps
@@ -121,8 +124,7 @@
{TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true
ZMQ-LINGER-MILLIS 0
TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS false
- TOPOLOGY-TRIDENT-BATCH-EMIT-INTERVAL-MILLIS 50
- }
+ TOPOLOGY-TRIDENT-BATCH-EMIT-INTERVAL-MILLIS 50}
daemon-conf
{STORM-CLUSTER-MODE "local"
STORM-ZOOKEEPER-PORT zk-port
@@ -130,8 +132,8 @@
nimbus-tmp (local-temp-path)
port-counter (mk-counter supervisor-slot-port-min)
nimbus (nimbus/service-handler
- (assoc daemon-conf STORM-LOCAL-DIR nimbus-tmp)
- (if inimbus inimbus (nimbus/standalone-nimbus)))
+ (assoc daemon-conf STORM-LOCAL-DIR nimbus-tmp)
+ (if inimbus inimbus (nimbus/standalone-nimbus)))
context (mk-shared-context daemon-conf)
cluster-map {:nimbus nimbus
:port-counter port-counter
@@ -147,13 +149,11 @@
(repeat supervisors {}))]
(doseq [sc supervisor-confs]
(add-supervisor cluster-map :ports ports-per-supervisor :conf sc))
- cluster-map
- ))
+ cluster-map))
(defn get-supervisor [cluster-map supervisor-id]
(let [finder-fn #(= (.get-id %) supervisor-id)]
- (find-first finder-fn @(:supervisors cluster-map))
- ))
+ (find-first finder-fn @(:supervisors cluster-map))))
(defn kill-supervisor [cluster-map supervisor-id]
(let [finder-fn #(= (.get-id %) supervisor-id)
@@ -162,8 +162,7 @@
supervisors)]
;; tmp-dir will be taken care of by shutdown
(reset! (:supervisors cluster-map) (remove-first finder-fn supervisors))
- (.shutdown sup)
- ))
+ (.shutdown sup)))
(defn kill-local-storm-cluster [cluster-map]
(.shutdown (:nimbus cluster-map))
@@ -181,17 +180,17 @@
(log-message "Deleting temporary path " t)
(try
(rmr t)
- (catch Exception e (log-message (.getMessage e)))) ;; on windows, the host process still holds lock on the logfile
- ))
+ ;; on windows, the host process still holds lock on the logfile
+ (catch Exception e (log-message (.getMessage e)))) ))
(def TEST-TIMEOUT-MS 5000)
(defmacro while-timeout [timeout-ms condition & body]
`(let [end-time# (+ (System/currentTimeMillis) ~timeout-ms)]
- (while ~condition
- (when (> (System/currentTimeMillis) end-time#)
- (throw (AssertionError. (str "Test timed out (" ~timeout-ms "ms)"))))
- ~@body)))
+ (while ~condition
+ (when (> (System/currentTimeMillis) end-time#)
+ (throw (AssertionError. (str "Test timed out (" ~timeout-ms "ms)"))))
+ ~@body)))
(defn wait-until-cluster-waiting
"Wait until the cluster is idle. Should be used with time simulation."
@@ -202,60 +201,59 @@
daemons (concat
[(:nimbus cluster-map)]
supervisors
- workers) ; because a worker may already be dead
- ]
+ ; because a worker may already be dead
+ workers)]
(while-timeout TEST-TIMEOUT-MS (not (every? (memfn waiting?) daemons))
- (Thread/sleep 10)
-;; (doseq [d daemons]
-;; (if-not ((memfn waiting?) d)
-;; (println d)))
- )))
+ (Thread/sleep 10)
+ ;; (doseq [d daemons]
+ ;; (if-not ((memfn waiting?) d)
+ ;; (println d)))
+ )))
(defn advance-cluster-time
([cluster-map secs increment-secs]
- (loop [left secs]
- (when (> left 0)
- (let [diff (min left increment-secs)]
- (advance-time-secs! diff)
- (wait-until-cluster-waiting cluster-map)
- (recur (- left diff))
- ))))
+ (loop [left secs]
+ (when (> left 0)
+ (let [diff (min left increment-secs)]
+ (advance-time-secs! diff)
+ (wait-until-cluster-waiting cluster-map)
+ (recur (- left diff))))))
([cluster-map secs]
- (advance-cluster-time cluster-map secs 1)
- ))
+ (advance-cluster-time cluster-map secs 1)))
-(defmacro with-local-cluster [[cluster-sym & args] & body]
+(defmacro with-local-cluster
+ [[cluster-sym & args] & body]
`(let [~cluster-sym (mk-local-storm-cluster ~@args)]
(try
~@body
- (catch Throwable t#
- (log-error t# "Error in cluster")
- (throw t#)
- )
- (finally
- (kill-local-storm-cluster ~cluster-sym)))
- ))
-
-(defmacro with-simulated-time-local-cluster [& args]
+ (catch Throwable t#
+ (log-error t# "Error in cluster")
+ (throw t#))
+ (finally
+ (kill-local-storm-cluster ~cluster-sym)))))
+
+(defmacro with-simulated-time-local-cluster
+ [& args]
`(with-simulated-time
- (with-local-cluster ~@args)))
+ (with-local-cluster ~@args)))
-(defmacro with-inprocess-zookeeper [port-sym & body]
+(defmacro with-inprocess-zookeeper
+ [port-sym & body]
`(with-local-tmp [tmp#]
- (let [[~port-sym zks#] (zk/mk-inprocess-zookeeper tmp#)]
- (try
- ~@body
- (finally
- (zk/shutdown-inprocess-zookeeper zks#)
- ))
- )))
-
-(defn submit-local-topology [nimbus storm-name conf topology]
+ (let [[~port-sym zks#] (zk/mk-inprocess-zookeeper tmp#)]
+ (try
+ ~@body
+ (finally
+ (zk/shutdown-inprocess-zookeeper zks#))))))
+
+(defn submit-local-topology
+ [nimbus storm-name conf topology]
(when-not (Utils/isValidConf conf)
(throw (IllegalArgumentException. "Topology conf is not json-serializable")))
(.submitTopology nimbus storm-name nil (to-json conf) topology))
-(defn submit-local-topology-with-opts [nimbus storm-name conf topology submit-opts]
+(defn submit-local-topology-with-opts
+ [nimbus storm-name conf topology submit-opts]
(when-not (Utils/isValidConf conf)
(throw (IllegalArgumentException. "Topology conf is not json-serializable")))
(.submitTopologyWithOpts nimbus storm-name nil (to-json conf) topology submit-opts))
@@ -269,141 +267,148 @@
new-assignments (assoc existing-assignments topology-id executor->node+port)]
new-assignments)))
-(defn submit-mocked-assignment [nimbus storm-name conf topology task->component executor->node+port]
+(defn submit-mocked-assignment
+ [nimbus storm-name conf topology task->component executor->node+port]
(with-var-roots [common/storm-task-info (fn [& ignored] task->component)
nimbus/compute-new-topology->executor->node+port (mocked-compute-new-topology->executor->node+port
- storm-name
- executor->node+port)]
- (submit-local-topology nimbus storm-name conf topology)
- ))
+ storm-name
+ executor->node+port)]
+ (submit-local-topology nimbus storm-name conf topology)))
(defn mk-capture-launch-fn [capture-atom]
(fn [supervisor storm-id port worker-id]
(let [supervisor-id (:supervisor-id supervisor)
existing (get @capture-atom [supervisor-id port] [])]
- (swap! capture-atom assoc [supervisor-id port] (conj existing storm-id))
- )))
+ (swap! capture-atom assoc [supervisor-id port] (conj existing storm-id)))))
-(defn find-worker-id [supervisor-conf port]
+(defn find-worker-id
+ [supervisor-conf port]
(let [supervisor-state (supervisor-state supervisor-conf)
worker->port (.get supervisor-state common/LS-APPROVED-WORKERS)]
- (first ((reverse-map worker->port) port))
- ))
+ (first ((reverse-map worker->port) port))))
-(defn find-worker-port [supervisor-conf worker-id]
+(defn find-worker-port
+ [supervisor-conf worker-id]
(let [supervisor-state (supervisor-state supervisor-conf)
- worker->port (.get supervisor-state common/LS-APPROVED-WORKERS)
- ]
- (worker->port worker-id)
- ))
+ worker->port (.get supervisor-state common/LS-APPROVED-WORKERS)]
+ (worker->port worker-id)))
-(defn mk-capture-shutdown-fn [capture-atom]
+(defn mk-capture-shutdown-fn
+ [capture-atom]
(let [existing-fn supervisor/shutdown-worker]
(fn [supervisor worker-id]
(let [conf (:conf supervisor)
supervisor-id (:supervisor-id supervisor)
port (find-worker-port conf worker-id)
- existing (get @capture-atom [supervisor-id port] 0)]
+ existing (get @capture-atom [supervisor-id port] 0)]
(swap! capture-atom assoc [supervisor-id port] (inc existing))
- (existing-fn supervisor worker-id)
- ))))
+ (existing-fn supervisor worker-id)))))
-(defmacro capture-changed-workers [& body]
+(defmacro capture-changed-workers
+ [& body]
`(let [launch-captured# (atom {})
shutdown-captured# (atom {})]
- (with-var-roots [supervisor/launch-worker (mk-capture-launch-fn launch-captured#)
- supervisor/shutdown-worker (mk-capture-shutdown-fn shutdown-captured#)]
- ~@body
- {:launched @launch-captured#
- :shutdown @shutdown-captured#}
- )))
-
-(defmacro capture-launched-workers [& body]
+ (with-var-roots [supervisor/launch-worker (mk-capture-launch-fn launch-captured#)
+ supervisor/shutdown-worker (mk-capture-shutdown-fn shutdown-captured#)]
+ ~@body
+ {:launched @launch-captured#
+ :shutdown @shutdown-captured#})))
+
+(defmacro capture-launched-workers
+ [& body]
`(:launched (capture-changed-workers ~@body)))
-(defmacro capture-shutdown-workers [& body]
+(defmacro capture-shutdown-workers
+ [& body]
`(:shutdown (capture-changed-workers ~@body)))
-(defnk aggregated-stat [cluster-map storm-name stat-key :component-ids nil]
+(defnk aggregated-stat
+ [cluster-map storm-name stat-key :component-ids nil]
(let [state (:storm-cluster-state cluster-map)
nimbus (:nimbus cluster-map)
storm-id (common/get-storm-id state storm-name)
-
component->tasks (reverse-map
- (common/storm-task-info
- (.getUserTopology nimbus storm-id)
- (from-json (.getTopologyConf nimbus storm-id))))
+ (common/storm-task-info
+ (.getUserTopology nimbus storm-id)
+ (from-json (.getTopologyConf nimbus storm-id))))
component->tasks (if component-ids
(select-keys component->tasks component-ids)
component->tasks)
task-ids (apply concat (vals component->tasks))
- assignment (.assignment-info state storm-id nil)
+ assignment (.assignment-info state storm-id nil)
taskbeats (.taskbeats state storm-id (:task->node+port assignment))
heartbeats (dofor [id task-ids] (get taskbeats id))
stats (dofor [hb heartbeats] (if hb (stat-key (:stats hb)) 0))]
- (reduce + stats)
- ))
+ (reduce + stats)))
-(defn emitted-spout-tuples [cluster-map topology storm-name]
- (aggregated-stat cluster-map
- storm-name
- :emitted
- :component-ids (keys (.get_spouts topology))))
+(defn emitted-spout-tuples
+ [cluster-map topology storm-name]
+ (aggregated-stat
+ cluster-map
+ storm-name
+ :emitted
+ :component-ids (keys (.get_spouts topology))))
-(defn transferred-tuples [cluster-map storm-name]
+(defn transferred-tuples
+ [cluster-map storm-name]
(aggregated-stat cluster-map storm-name :transferred))
-(defn acked-tuples [cluster-map storm-name]
+(defn acked-tuples
+ [cluster-map storm-name]
(aggregated-stat cluster-map storm-name :acked))
-(defn simulate-wait [cluster-map]
+(defn simulate-wait
+ [cluster-map]
(if (Time/isSimulating)
(advance-cluster-time cluster-map 10)
- (Thread/sleep 100)
- ))
+ (Thread/sleep 100)))
(defprotocol CompletableSpout
- (exhausted? [this] "Whether all the tuples for this spout have been completed.")
- (cleanup [this] "Cleanup any global state kept")
- (startup [this] "Prepare the spout (globally) before starting the topology"))
+ (exhausted?
+ [this]
+ "Whether all the tuples for this spout have been completed.")
+ (cleanup
+ [this]
+ "Cleanup any global state kept")
+ (startup
+ [this]
+ "Prepare the spout (globally) before starting the topology"))
(extend-type FixedTupleSpout
CompletableSpout
(exhausted? [this]
- (= (-> this .getSourceTuples count)
- (.getCompleted this)))
+ (= (-> this .getSourceTuples count)
+ (.getCompleted this)))
(cleanup [this]
- (.cleanup this))
- (startup [this]
- ))
+ (.cleanup this))
+ (startup [this]))
(extend-type TransactionalSpoutCoordinator
CompletableSpout
(exhausted? [this]
- (exhausted? (.getSpout this)))
+ (exhausted? (.getSpout this)))
(cleanup [this]
- (cleanup (.getSpout this)))
+ (cleanup (.getSpout this)))
(startup [this]
- (startup (.getSpout this))))
+ (startup (.getSpout this))))
(extend-type PartitionedTransactionalSpoutExecutor
CompletableSpout
(exhausted? [this]
- (exhausted? (.getPartitionedSpout this)))
+ (exhausted? (.getPartitionedSpout this)))
(cleanup [this]
- (cleanup (.getPartitionedSpout this)))
+ (cleanup (.getPartitionedSpout this)))
(startup [this]
- (startup (.getPartitionedSpout this))
- ))
+ (startup (.getPartitionedSpout this))))
(extend-type MemoryTransactionalSpout
CompletableSpout
(exhausted? [this]
- (.isExhaustedTuples this))
+ (.isExhaustedTuples this))
(cleanup [this]
- (.cleanup this))
+ (.cleanup this))
(startup [this]
- (.startup this)))
+ (.startup this)))
(defn spout-objects [spec-map]
(for [[_ spout-spec] spec-map]
@@ -411,7 +416,8 @@
.get_spout_object
deserialized-component-object)))
-(defn capture-topology [topology]
+(defn capture-topology
+ [topology]
(let [topology (.deepCopy topology)
spouts (.get_spouts topology)
bolts (.get_bolts topology)
@@ -424,21 +430,24 @@
(.set_bolts topology
(assoc (clojurify-structure bolts)
(uuid)
- (Bolt.
- (serialize-component-object capturer)
- (mk-plain-component-common (into {} (for [[id direct?] all-streams]
- [id (if direct?
- (mk-direct-grouping)
- (mk-global-grouping))]))
- {}
- nil))
- ))
+ (Bolt.
+ (serialize-component-object capturer)
+ (mk-plain-component-common (into {} (for [[id direct?] all-streams]
+ [id (if direct?
+ (mk-direct-grouping)
+ (mk-global-grouping))]))
+ {}
+ nil))))
{:topology topology
- :capturer capturer}
- ))
+ :capturer capturer}))
;; TODO: mock-sources needs to be able to mock out state spouts as well
-(defnk complete-topology [cluster-map topology :mock-sources {} :storm-conf {} :cleanup-state true :topology-name nil]
+(defnk complete-topology
+ [cluster-map topology
+ :mock-sources {}
+ :storm-conf {}
+ :cleanup-state true
+ :topology-name nil]
;; TODO: the idea of mocking for transactional topologies should be done an
;; abstraction level above... should have a complete-transactional-topology for this
(let [{topology :topology capturer :capturer} (capture-topology topology)
@@ -447,153 +456,144 @@
spouts (.get_spouts topology)
replacements (map-val (fn [v]
(FixedTupleSpout.
- (for [tup v]
- (if (map? tup)
- (FixedTuple. (:stream tup) (:values tup))
- tup))))
- mock-sources)
-
-
- ]
+ (for [tup v]
+ (if (map? tup)
+ (FixedTuple. (:stream tup) (:values tup))
+ tup))))
+ mock-sources)]
(doseq [[id spout] replacements]
(let [spout-spec (get spouts id)]
- (.set_spout_object spout-spec (serialize-component-object spout))
- ))
+ (.set_spout_object spout-spec (serialize-component-object spout))))
(doseq [spout (spout-objects spouts)]
(when-not (extends? CompletableSpout (.getClass spout))
- (throw (RuntimeException. "Cannot complete topology unless every spout is a CompletableSpout (or mocked to be)"))
- ))
+ (throw (RuntimeException. "Cannot complete topology unless every spout is a CompletableSpout (or mocked to be)"))))
(doseq [spout (spout-objects spouts)]
(startup spout))
-
+
(submit-local-topology (:nimbus cluster-map) storm-name storm-conf topology)
-
-
+
(let [storm-id (common/get-storm-id state storm-name)]
(while-timeout TEST-TIMEOUT-MS (not (every? exhausted? (spout-objects spouts)))
- (simulate-wait cluster-map))
+ (simulate-wait cluster-map))
(.killTopologyWithOpts (:nimbus cluster-map) storm-name (doto (KillOptions.) (.set_wait_secs 0)))
(while-timeout TEST-TIMEOUT-MS (.assignment-info state storm-id nil)
- (simulate-wait cluster-map))
+ (simulate-wait cluster-map))
(when cleanup-state
(doseq [spout (spout-objects spouts)]
(cleanup spout))))
(if cleanup-state
(.getAndRemoveResults capturer)
- (.getAndClearResults capturer))
- ))
+ (.getAndClearResults capturer))))
(defn read-tuples
([results component-id stream-id]
- (let [fixed-tuples (get results component-id [])]
- (mapcat
- (fn [ft]
- (if (= stream-id (. ft stream))
- [(vec (. ft values))]))
- fixed-tuples)
- ))
- ([results component-id]
- (read-tuples results component-id Utils/DEFAULT_STREAM_ID)
+ (let [fixed-tuples (get results component-id [])]
+ (mapcat
+ (fn [ft]
+ (if (= stream-id (. ft stream))
+ [(vec (. ft values))]))
+ fixed-tuples)
))
+ ([results component-id]
+ (read-tuples results component-id Utils/DEFAULT_STREAM_ID)))
-(defn ms= [& args]
+(defn ms=
+ [& args]
(apply = (map multi-set args)))
(def TRACKER-BOLT-ID "+++tracker-bolt")
-;; TODO: should override system-topology! and wrap everything there
+;; TODO: should override system-topology! and wrap everything there
(defn mk-tracked-topology
([tracked-cluster topology]
- (let [track-id (::track-id tracked-cluster)
- ret (.deepCopy topology)]
- (dofor [[_ bolt] (.get_bolts ret)
- :let [obj (deserialized-component-object (.get_bolt_object bolt))]]
- (.set_bolt_object bolt (serialize-component-object
- (BoltTracker. obj track-id))))
- (dofor [[_ spout] (.get_spouts ret)
- :let [obj (deserialized-component-object (.get_spout_object spout))]]
- (.set_spout_object spout (serialize-component-object
- (SpoutTracker. obj track-id))))
- {:topology ret
- :last-spout-emit (atom 0)
- :cluster tracked-cluster
- }
- )))
-
-(defn assoc-track-id [cluster track-id]
+ (let [track-id (::track-id tracked-cluster)
+ ret (.deepCopy topology)]
+ (dofor [[_ bolt] (.get_bolts ret)
+ :let [obj (deserialized-component-object (.get_bolt_object bolt))]]
+ (.set_bolt_object bolt (serialize-component-object
+ (BoltTracker. obj track-id))))
+ (dofor [[_ spout] (.get_spouts ret)
+ :let [obj (deserialized-component-object (.get_spout_object spout))]]
+ (.set_spout_object spout (serialize-component-object
+ (SpoutTracker. obj track-id))))
+ {:topology ret
+ :last-spout-emit (atom 0)
+ :cluster tracked-cluster})))
+
+(defn assoc-track-id
+ [cluster track-id]
(assoc cluster ::track-id track-id))
-(defn increment-global! [id key amt]
+(defn increment-global!
+ [id key amt]
(-> (RegisteredGlobalState/getState id)
(get key)
(.addAndGet amt)))
-(defn global-amt [id key]
+(defn global-amt
+ [id key]
(-> (RegisteredGlobalState/getState id)
(get key)
- .get
- ))
+ .get))
-(defmacro with-tracked-cluster [[cluster-sym & cluster-args] & body]
+(defmacro with-tracked-cluster
+ [[cluster-sym & cluster-args] & body]
`(let [id# (uuid)]
- (RegisteredGlobalState/setState id#
- (doto (ConcurrentHashMap.)
- (.put "spout-emitted" (AtomicInteger. 0))
- (.put "transferred" (AtomicInteger. 0))
- (.put "processed" (AtomicInteger. 0))))
- (with-var-roots [acker/mk-acker-bolt (let [old# acker/mk-acker-bolt]
- (fn [& args#]
- (NonRichBoltTracker. (apply old# args#) id#)
- ))
- ;; critical that this particular function is overridden here,
- ;; since the transferred stat needs to be incremented at the moment
- ;; of tuple emission (and not on a separate thread later) for
- ;; topologies to be tracked correctly. This is because "transferred" *must*
- ;; be incremented before "processing".
- executor/mk-executor-transfer-fn
- (let [old# executor/mk-executor-transfer-fn]
- (fn [& args#]
- (let [transferrer# (apply old# args#)]
- (fn [& args2#]
- ;; (log-message "Transferring: " transfer-args#)
- (increment-global! id# "transferred" 1)
- (apply transferrer# args2#)
- ))))
- ]
+ (RegisteredGlobalState/setState
+ id#
+ (doto (ConcurrentHashMap.)
+ (.put "spout-emitted" (AtomicInteger. 0))
+ (.put "transferred" (AtomicInteger. 0))
+ (.put "processed" (AtomicInteger. 0))))
+ (with-var-roots
+ [acker/mk-acker-bolt
+ (let [old# acker/mk-acker-bolt]
+ (fn [& args#] (NonRichBoltTracker. (apply old# args#) id#)))
+ ;; critical that this particular function is overridden here,
+ ;; since the transferred stat needs to be incremented at the moment
+ ;; of tuple emission (and not on a separate thread later) for
+ ;; topologies to be tracked correctly. This is because "transferred" *must*
+ ;; be incremented before "processing".
+ executor/mk-executor-transfer-fn
+ (let [old# executor/mk-executor-transfer-fn]
+ (fn [& args#]
+ (let [transferrer# (apply old# args#)]
+ (fn [& args2#]
+ ;; (log-message "Transferring: " transfer-args#)
+ (increment-global! id# "transferred" 1)
+ (apply transferrer# args2#)))))]
(with-local-cluster [~cluster-sym ~@cluster-args]
- (let [~cluster-sym (assoc-track-id ~cluster-sym id#)]
- ~@body)
- ))
- (RegisteredGlobalState/clearState id#)
- ))
+ (let [~cluster-sym (assoc-track-id ~cluster-sym id#)]
+ ~@body)))
+ (RegisteredGlobalState/clearState id#)))
(defn tracked-wait
"Waits until topology is idle and 'amt' more tuples have been emitted by spouts."
([tracked-topology]
- (tracked-wait tracked-topology 1))
+ (tracked-wait tracked-topology 1))
([tracked-topology amt]
- (let [target (+ amt @(:last-spout-emit tracked-topology))
- track-id (-> tracked-topology :cluster ::track-id)
- waiting? (fn []
- (or (not= target (global-amt track-id "spout-emitted"))
- (not= (global-amt track-id "transferred")
- (global-amt track-id "processed"))
- ))]
- (while-timeout TEST-TIMEOUT-MS (waiting?)
- ;; (println "Spout emitted: " (global-amt track-id "spout-emitted"))
- ;; (println "Processed: " (global-amt track-id "processed"))
- ;; (println "Transferred: " (global-amt track-id "transferred"))
- (Thread/sleep 500))
- (reset! (:last-spout-emit tracked-topology) target)
- )))
-
-(defnk test-tuple [values
- :stream Utils/DEFAULT_STREAM_ID
- :component "component"
- :fields nil]
+ (let [target (+ amt @(:last-spout-emit tracked-topology))
+ track-id (-> tracked-topology :cluster ::track-id)
+ waiting? (fn []
+ (or (not= target (global-amt track-id "spout-emitted"))
+ (not= (global-amt track-id "transferred")
+ (global-amt track-id "processed"))
+ ))]
+ (while-timeout TEST-TIMEOUT-MS (waiting?)
+ ;; (println "Spout emitted: " (global-amt track-id "spout-emitted"))
+ ;; (println "Processed: " (global-amt track-id "processed"))
+ ;; (println "Transferred: " (global-amt track-id "transferred"))
+ (Thread/sleep 500))
+ (reset! (:last-spout-emit tracked-topology) target))))
+
+(defnk test-tuple
+ [values
+ :stream Utils/DEFAULT_STREAM_ID
+ :component "component"
+ :fields nil]
(let [fields (or fields
(->> (iterate inc 1)
(take (count values))
@@ -618,5 +618,4 @@
(HashMap.)
(HashMap.)
(atom false))]
- (TupleImpl. context values 1 stream)
- ))
+ (TupleImpl. context values 1 stream)))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2278fc96/storm-core/src/clj/backtype/storm/testing4j.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/testing4j.clj b/storm-core/src/clj/backtype/storm/testing4j.clj
index 5d44604..ff939a9 100644
--- a/storm-core/src/clj/backtype/storm/testing4j.clj
+++ b/storm-core/src/clj/backtype/storm/testing4j.clj
@@ -51,7 +51,6 @@
^:static [testTuple [java.util.List] backtype.storm.tuple.Tuple]
^:static [testTuple [java.util.List backtype.storm.testing.MkTupleParam] backtype.storm.tuple.Tuple]]))
-
(defn -completeTopology
([^ILocalCluster cluster ^StormTopology topology ^CompleteTopologyParam completeTopologyParam]
(let [mocked-sources (or (-> completeTopologyParam .getMockedSources .getData) {})
@@ -66,11 +65,13 @@
([^ILocalCluster cluster ^StormTopology topology]
(-completeTopology cluster topology (CompleteTopologyParam.))))
-(defn -withSimulatedTime [^Runnable code]
+(defn -withSimulatedTime
+ [^Runnable code]
(with-simulated-time
(.run code)))
-(defmacro with-cluster [cluster-type mkClusterParam code]
+(defmacro with-cluster
+ [cluster-type mkClusterParam code]
`(let [supervisors# (or (.getSupervisors ~mkClusterParam) 2)
ports-per-supervisor# (or (.getPortsPerSupervisor ~mkClusterParam) 3)
daemon-conf# (or (.getDaemonConf ~mkClusterParam) {})]
@@ -98,7 +99,8 @@
([^TestJob code]
(-withTrackedCluster (MkClusterParam.) code)))
-(defn- find-tuples [^List fixed-tuples ^String stream]
+(defn- find-tuples
+ [^List fixed-tuples ^String stream]
(let [ret (ArrayList.)]
(doseq [fixed-tuple fixed-tuples]
(if (= (.stream fixed-tuple) stream)
@@ -107,48 +109,52 @@
(defn -readTuples
([^Map result ^String componentId ^String streamId]
- (let [stream-result (.get result componentId)
- ret (if stream-result
- (find-tuples stream-result streamId)
- [])]
- ret))
+ (let [stream-result (.get result componentId)
+ ret (if stream-result
+ (find-tuples stream-result streamId)
+ [])]
+ ret))
([^Map result ^String componentId]
- (-readTuples result componentId Utils/DEFAULT_STREAM_ID)))
+ (-readTuples result componentId Utils/DEFAULT_STREAM_ID)))
-(defn -mkTrackedTopology [^ILocalCluster trackedCluster ^StormTopology topology]
+(defn -mkTrackedTopology
+ [^ILocalCluster trackedCluster ^StormTopology topology]
(-> (mk-tracked-topology (.getState trackedCluster) topology)
(TrackedTopology.)))
(defn -trackedWait
([^TrackedTopology trackedTopology ^Integer amt]
- (tracked-wait trackedTopology amt))
+ (tracked-wait trackedTopology amt))
([^TrackedTopology trackedTopology]
- (-trackedWait trackedTopology 1)))
+ (-trackedWait trackedTopology 1)))
(defn -advanceClusterTime
([^ILocalCluster cluster ^Integer secs ^Integer step]
- (advance-cluster-time (.getState cluster) secs step))
+ (advance-cluster-time (.getState cluster) secs step))
([^ILocalCluster cluster ^Integer secs]
- (-advanceClusterTime cluster secs 1)))
+ (-advanceClusterTime cluster secs 1)))
-(defn- multiseteq [^Object obj1 ^Object obj2]
+(defn- multiseteq
+ [^Object obj1 ^Object obj2]
(let [obj1 (clojurify-structure obj1)
obj2 (clojurify-structure obj2)]
(ms= obj1 obj2)))
-(defn -multiseteq [^Collection coll1 ^Collection coll2]
- (multiseteq coll1 coll2))
+(defn -multiseteq
+ [^Collection coll1 ^Collection coll2]
+ (multiseteq coll1 coll2))
-(defn -multiseteq [^Map coll1 ^Map coll2]
+(defn -multiseteq
+ [^Map coll1 ^Map coll2]
(multiseteq coll1 coll2))
(defn -testTuple
([^List values]
- (-testTuple values nil))
+ (-testTuple values nil))
([^List values ^MkTupleParam param]
- (if (nil? param)
- (test-tuple values)
- (let [stream (or (.getStream param) Utils/DEFAULT_STREAM_ID)
- component (or (.getComponent param) "component")
- fields (.getFields param)]
- (test-tuple values :stream stream :component component :fields fields)))))
\ No newline at end of file
+ (if (nil? param)
+ (test-tuple values)
+ (let [stream (or (.getStream param) Utils/DEFAULT_STREAM_ID)
+ component (or (.getComponent param) "component")
+ fields (.getFields param)]
+ (test-tuple values :stream stream :component component :fields fields)))))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2278fc96/storm-core/src/clj/backtype/storm/thrift.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/thrift.clj b/storm-core/src/clj/backtype/storm/thrift.clj
index ec8ce5d..f778d23 100644
--- a/storm-core/src/clj/backtype/storm/thrift.clj
+++ b/storm-core/src/clj/backtype/storm/thrift.clj
@@ -13,11 +13,14 @@
;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;; See the License for the specific language governing permissions and
;; limitations under the License.
+
(ns backtype.storm.thrift
(:import [java.util HashMap])
- (:import [backtype.storm.generated JavaObject Grouping Nimbus StormTopology StormTopology$_Fields
- Bolt Nimbus$Client Nimbus$Iface ComponentCommon Grouping$_Fields SpoutSpec NullStruct StreamInfo
- GlobalStreamId ComponentObject ComponentObject$_Fields ShellComponent])
+ (:import [backtype.storm.generated JavaObject Grouping Nimbus StormTopology
+ StormTopology$_Fields Bolt Nimbus$Client Nimbus$Iface
+ ComponentCommon Grouping$_Fields SpoutSpec NullStruct StreamInfo
+ GlobalStreamId ComponentObject ComponentObject$_Fields
+ ShellComponent])
(:import [backtype.storm.utils Utils])
(:import [backtype.storm Constants])
(:import [backtype.storm.grouping CustomStreamGrouping])
@@ -25,14 +28,13 @@
(:import [backtype.storm.clojure RichShellBolt RichShellSpout])
(:import [org.apache.thrift.protocol TBinaryProtocol TProtocol])
(:import [org.apache.thrift.transport TTransport TFramedTransport TSocket])
- (:use [backtype.storm util config log])
- )
+ (:use [backtype.storm util config log]))
-(defn instantiate-java-object [^JavaObject obj]
+(defn instantiate-java-object
+ [^JavaObject obj]
(let [name (symbol (.get_full_class_name obj))
args (map (memfn getFieldValue) (.get_args_list obj))]
- (eval `(new ~name ~@args))
- ))
+ (eval `(new ~name ~@args))))
(def grouping-constants
{Grouping$_Fields/FIELDS :fields
@@ -42,144 +44,184 @@
Grouping$_Fields/CUSTOM_SERIALIZED :custom-serialized
Grouping$_Fields/CUSTOM_OBJECT :custom-object
Grouping$_Fields/DIRECT :direct
- Grouping$_Fields/LOCAL_OR_SHUFFLE :local-or-shuffle
- })
+ Grouping$_Fields/LOCAL_OR_SHUFFLE :local-or-shuffle})
-(defn grouping-type [^Grouping grouping]
+(defn grouping-type
+ [^Grouping grouping]
(grouping-constants (.getSetField grouping)))
-(defn field-grouping [^Grouping grouping]
+(defn field-grouping
+ [^Grouping grouping]
(when-not (= (grouping-type grouping) :fields)
(throw (IllegalArgumentException. "Tried to get grouping fields from non fields grouping")))
(.get_fields grouping))
-(defn global-grouping? [^Grouping grouping]
+(defn global-grouping?
+ [^Grouping grouping]
(and (= :fields (grouping-type grouping))
- (empty? (field-grouping grouping))
- ))
+ (empty? (field-grouping grouping))))
-(defn parallelism-hint [^ComponentCommon component-common]
+(defn parallelism-hint
+ [^ComponentCommon component-common]
(let [phint (.get_parallelism_hint component-common)]
- (if-not (.is_set_parallelism_hint component-common) 1 phint)
- ))
+ (if-not (.is_set_parallelism_hint component-common) 1 phint)))
-(defn nimbus-client-and-conn [host port]
+(defn nimbus-client-and-conn
+ [host port]
(log-message "Connecting to Nimbus at " host ":" port)
(let [transport (TFramedTransport. (TSocket. host port))
prot (TBinaryProtocol. transport)
client (Nimbus$Client. prot)]
- (.open transport)
- [client transport] ))
-
-(defmacro with-nimbus-connection [[client-sym host port] & body]
- `(let [[^Nimbus$Client ~client-sym ^TTransport conn#] (nimbus-client-and-conn ~host ~port)]
- (try
- ~@body
- (finally (.close conn#)))
- ))
-
-(defmacro with-configured-nimbus-connection [client-sym & body]
+ (.open transport)
+ [client transport]))
+
+(defmacro with-nimbus-connection
+ [[client-sym host port] & body]
+ `(let [[^Nimbus$Client ~client-sym ^TTransport conn#]
+ (nimbus-client-and-conn ~host ~port)]
+ (try
+ ~@body
+ (finally (.close conn#)))))
+
+(defmacro with-configured-nimbus-connection
+ [client-sym & body]
`(let [conf# (read-storm-config)
host# (conf# NIMBUS-HOST)
port# (conf# NIMBUS-THRIFT-PORT)]
(with-nimbus-connection [~client-sym host# port#]
~@body )))
-(defn direct-output-fields [fields]
+(defn direct-output-fields
+ [fields]
(StreamInfo. fields true))
-(defn output-fields [fields]
+(defn output-fields
+ [fields]
(StreamInfo. fields false))
-(defn mk-output-spec [output-spec]
- (let [output-spec (if (map? output-spec) output-spec {Utils/DEFAULT_STREAM_ID output-spec})]
+(defn mk-output-spec
+ [output-spec]
+ (let [output-spec (if (map? output-spec)
+ output-spec
+ {Utils/DEFAULT_STREAM_ID output-spec})]
(map-val
(fn [out]
(if (instance? StreamInfo out)
out
- (StreamInfo. out false)
- ))
- output-spec
- )))
+ (StreamInfo. out false)))
+ output-spec)))
-(defnk mk-plain-component-common [inputs output-spec parallelism-hint :conf nil]
+(defnk mk-plain-component-common
+ [inputs output-spec parallelism-hint :conf nil]
(let [ret (ComponentCommon. (HashMap. inputs) (HashMap. (mk-output-spec output-spec)))]
(when parallelism-hint
(.set_parallelism_hint ret parallelism-hint))
(when conf
(.set_json_conf ret (to-json conf)))
- ret
- ))
+ ret))
-(defnk mk-spout-spec* [spout outputs :p nil :conf nil]
+(defnk mk-spout-spec*
+ [spout outputs :p nil :conf nil]
(SpoutSpec. (ComponentObject/serialized_java (Utils/serialize spout))
(mk-plain-component-common {} outputs p :conf conf)))
-(defn mk-shuffle-grouping []
+(defn mk-shuffle-grouping
+ []
(Grouping/shuffle (NullStruct.)))
-(defn mk-local-or-shuffle-grouping []
+(defn mk-local-or-shuffle-grouping
+ []
(Grouping/local_or_shuffle (NullStruct.)))
-(defn mk-fields-grouping [fields]
+(defn mk-fields-grouping
+ [fields]
(Grouping/fields fields))
-(defn mk-global-grouping []
+(defn mk-global-grouping
+ []
(mk-fields-grouping []))
-(defn mk-direct-grouping []
+(defn mk-direct-grouping
+ []
(Grouping/direct (NullStruct.)))
-(defn mk-all-grouping []
+(defn mk-all-grouping
+ []
(Grouping/all (NullStruct.)))
-(defn mk-none-grouping []
+(defn mk-none-grouping
+ []
(Grouping/none (NullStruct.)))
-(defn deserialized-component-object [^ComponentObject obj]
+(defn deserialized-component-object
+ [^ComponentObject obj]
(when (not= (.getSetField obj) ComponentObject$_Fields/SERIALIZED_JAVA)
(throw (RuntimeException. "Cannot deserialize non-java-serialized object")))
- (Utils/deserialize (.get_serialized_java obj))
- )
+ (Utils/deserialize (.get_serialized_java obj)))
-(defn serialize-component-object [obj]
+(defn serialize-component-object
+ [obj]
(ComponentObject/serialized_java (Utils/serialize obj)))
-(defn- mk-grouping [grouping-spec]
- (cond (nil? grouping-spec) (mk-none-grouping)
- (instance? Grouping grouping-spec) grouping-spec
- (instance? CustomStreamGrouping grouping-spec) (Grouping/custom_serialized (Utils/serialize grouping-spec))
- (instance? JavaObject grouping-spec) (Grouping/custom_object grouping-spec)
- (sequential? grouping-spec) (mk-fields-grouping grouping-spec)
- (= grouping-spec :shuffle) (mk-shuffle-grouping)
- (= grouping-spec :local-or-shuffle) (mk-local-or-shuffle-grouping)
- (= grouping-spec :none) (mk-none-grouping)
- (= grouping-spec :all) (mk-all-grouping)
- (= grouping-spec :global) (mk-global-grouping)
- (= grouping-spec :direct) (mk-direct-grouping)
- true (throw (IllegalArgumentException. (str grouping-spec " is not a valid grouping")))
- ))
-
-(defn- mk-inputs [inputs]
- (into {}
- (for [[stream-id grouping-spec] inputs]
- [(if (sequential? stream-id)
- (GlobalStreamId. (first stream-id) (second stream-id))
- (GlobalStreamId. stream-id Utils/DEFAULT_STREAM_ID))
- (mk-grouping grouping-spec)]
- )))
-
-(defnk mk-bolt-spec* [inputs bolt outputs :p nil :conf nil]
+(defn- mk-grouping
+ [grouping-spec]
+ (cond (nil? grouping-spec)
+ (mk-none-grouping)
+
+ (instance? Grouping grouping-spec)
+ grouping-spec
+
+ (instance? CustomStreamGrouping grouping-spec)
+ (Grouping/custom_serialized (Utils/serialize grouping-spec))
+
+ (instance? JavaObject grouping-spec)
+ (Grouping/custom_object grouping-spec)
+
+ (sequential? grouping-spec)
+ (mk-fields-grouping grouping-spec)
+
+ (= grouping-spec :shuffle)
+ (mk-shuffle-grouping)
+
+ (= grouping-spec :local-or-shuffle)
+ (mk-local-or-shuffle-grouping)
+ (= grouping-spec :none)
+ (mk-none-grouping)
+
+ (= grouping-spec :all)
+ (mk-all-grouping)
+
+ (= grouping-spec :global)
+ (mk-global-grouping)
+
+ (= grouping-spec :direct)
+ (mk-direct-grouping)
+
+ true
+ (throw (IllegalArgumentException.
+ (str grouping-spec " is not a valid grouping")))))
+
+(defn- mk-inputs
+ [inputs]
+ (into {} (for [[stream-id grouping-spec] inputs]
+ [(if (sequential? stream-id)
+ (GlobalStreamId. (first stream-id) (second stream-id))
+ (GlobalStreamId. stream-id Utils/DEFAULT_STREAM_ID))
+ (mk-grouping grouping-spec)])))
+
+(defnk mk-bolt-spec*
+ [inputs bolt outputs :p nil :conf nil]
(let [common (mk-plain-component-common (mk-inputs inputs) outputs p :conf conf)]
(Bolt. (ComponentObject/serialized_java (Utils/serialize bolt))
- common )))
+ common)))
-(defnk mk-spout-spec [spout :parallelism-hint nil :p nil :conf nil]
+(defnk mk-spout-spec
+ [spout :parallelism-hint nil :p nil :conf nil]
(let [parallelism-hint (if p p parallelism-hint)]
- {:obj spout :p parallelism-hint :conf conf}
- ))
+ {:obj spout :p parallelism-hint :conf conf}))
-(defn- shell-component-params [command script-or-output-spec kwargs]
+(defn- shell-component-params
+ [command script-or-output-spec kwargs]
(if (string? script-or-output-spec)
[(into-array String [command script-or-output-spec])
(first kwargs)
@@ -188,41 +230,46 @@
script-or-output-spec
kwargs]))
-(defnk mk-bolt-spec [inputs bolt :parallelism-hint nil :p nil :conf nil]
+(defnk mk-bolt-spec
+ [inputs bolt :parallelism-hint nil :p nil :conf nil]
(let [parallelism-hint (if p p parallelism-hint)]
- {:obj bolt :inputs inputs :p parallelism-hint :conf conf}
- ))
+ {:obj bolt :inputs inputs :p parallelism-hint :conf conf}))
-(defn mk-shell-bolt-spec [inputs command script-or-output-spec & kwargs]
+(defn mk-shell-bolt-spec
+ [inputs command script-or-output-spec & kwargs]
(let [[command output-spec kwargs]
(shell-component-params command script-or-output-spec kwargs)]
- (apply mk-bolt-spec inputs (RichShellBolt. command (mk-output-spec output-spec)) kwargs)))
+ (apply mk-bolt-spec inputs
+ (RichShellBolt. command (mk-output-spec output-spec)) kwargs)))
-(defn mk-shell-spout-spec [command script-or-output-spec & kwargs]
+(defn mk-shell-spout-spec
+ [command script-or-output-spec & kwargs]
(let [[command output-spec kwargs]
(shell-component-params command script-or-output-spec kwargs)]
- (apply mk-spout-spec (RichShellSpout. command (mk-output-spec output-spec)) kwargs)))
+ (apply mk-spout-spec
+ (RichShellSpout. command (mk-output-spec output-spec)) kwargs)))
-(defn- add-inputs [declarer inputs]
+(defn- add-inputs
+ [declarer inputs]
(doseq [[id grouping] (mk-inputs inputs)]
- (.grouping declarer id grouping)
- ))
+ (.grouping declarer id grouping)))
(defn mk-topology
([spout-map bolt-map]
- (let [builder (TopologyBuilder.)]
- (doseq [[name {spout :obj p :p conf :conf}] spout-map]
- (-> builder (.setSpout name spout (if-not (nil? p) (int p) p)) (.addConfigurations conf)))
- (doseq [[name {bolt :obj p :p conf :conf inputs :inputs}] bolt-map]
- (-> builder (.setBolt name bolt (if-not (nil? p) (int p) p)) (.addConfigurations conf) (add-inputs inputs)))
- (.createTopology builder)
- ))
+ (let [builder (TopologyBuilder.)]
+ (doseq [[name {spout :obj p :p conf :conf}] spout-map]
+ (-> builder (.setSpout name spout (if-not (nil? p) (int p) p)) (.addConfigurations conf)))
+ (doseq [[name {bolt :obj p :p conf :conf inputs :inputs}] bolt-map]
+ (-> builder (.setBolt name bolt (if-not (nil? p) (int p) p)) (.addConfigurations conf) (add-inputs inputs)))
+ (.createTopology builder)))
([spout-map bolt-map state-spout-map]
- (mk-topology spout-map bolt-map)))
+ (mk-topology spout-map bolt-map)))
;; clojurify-structure is needed or else every element becomes the same after successive calls
;; don't know why this happens
-(def STORM-TOPOLOGY-FIELDS (-> StormTopology/metaDataMap clojurify-structure keys))
+(def STORM-TOPOLOGY-FIELDS
+ (-> StormTopology/metaDataMap clojurify-structure keys))
-(def SPOUT-FIELDS [StormTopology$_Fields/SPOUTS
- StormTopology$_Fields/STATE_SPOUTS])
+(def SPOUT-FIELDS
+ [StormTopology$_Fields/SPOUTS
+ StormTopology$_Fields/STATE_SPOUTS])
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2278fc96/storm-core/src/clj/backtype/storm/timer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/timer.clj b/storm-core/src/clj/backtype/storm/timer.clj
index 78b1f1c..b105698 100644
--- a/storm-core/src/clj/backtype/storm/timer.clj
+++ b/storm-core/src/clj/backtype/storm/timer.clj
@@ -13,57 +13,55 @@
;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;; See the License for the specific language governing permissions and
;; limitations under the License.
+
(ns backtype.storm.timer
(:import [backtype.storm.utils Time])
(:import [java.util PriorityQueue Comparator])
(:import [java.util.concurrent Semaphore])
- (:use [backtype.storm util log])
- )
+ (:use [backtype.storm util log]))
-;; The timer defined in this file is very similar to java.util.Timer, except it integrates with
-;; Storm's time simulation capabilities. This lets us test code that does asynchronous work on the timer thread
+;; The timer defined in this file is very similar to java.util.Timer, except
+;; it integrates with Storm's time simulation capabilities. This lets us test
+;; code that does asynchronous work on the timer thread
-(defnk mk-timer [:kill-fn (fn [& _] )]
+(defnk mk-timer
+ [:kill-fn (fn [& _] )]
(let [queue (PriorityQueue. 10
(reify Comparator
(compare [this o1 o2]
- (- (first o1) (first o2))
- )
+ (- (first o1) (first o2)))
(equals [this obj]
- true
- )))
+ true)))
active (atom true)
lock (Object.)
notifier (Semaphore. 0)
timer-thread (Thread.
- (fn []
- (while @active
- (try
- (let [[time-millis _ _ :as elem] (locking lock (.peek queue))]
- (if (and elem (>= (current-time-millis) time-millis))
- ;; imperative to not run the function inside the timer lock
- ;; otherwise, it's possible to deadlock if function deals with other locks
- ;; (like the submit lock)
- (let [afn (locking lock (second (.poll queue)))]
- (afn))
- (if time-millis ;; if any events are scheduled
- ;; sleep until event generation
- ;; note that if any recurring events are scheduled then we will always go through
- ;; this branch, sleeping only the exact necessary amount of time
- (Time/sleep (- time-millis (current-time-millis)))
- ;; else poll to see if any new event was scheduled
- ;; this is in essence the response time for detecting any new event schedulings when
- ;; there are no scheduled events
- (Time/sleep 1000))
- ))
- (catch Throwable t
- ;; because the interrupted exception can be wrapped in a runtimeexception
- (when-not (exception-cause? InterruptedException t)
- (kill-fn t)
- (reset! active false)
- (throw t))
- )))
- (.release notifier)))]
+ (fn []
+ (while @active
+ (try
+ (let [[time-millis _ _ :as elem] (locking lock (.peek queue))]
+ (if (and elem (>= (current-time-millis) time-millis))
+ ;; imperative to not run the function inside the timer lock
+ ;; otherwise, it's possible to deadlock if function deals with other locks
+ ;; (like the submit lock)
+ (let [afn (locking lock (second (.poll queue)))]
+ (afn))
+ (if time-millis ;; if any events are scheduled
+ ;; sleep until event generation
+ ;; note that if any recurring events are scheduled then we will always go through
+ ;; this branch, sleeping only the exact necessary amount of time
+ (Time/sleep (- time-millis (current-time-millis)))
+ ;; else poll to see if any new event was scheduled
+ ;; this is in essence the response time for detecting any new event schedulings when
+ ;; there are no scheduled events
+ (Time/sleep 1000))))
+ (catch Throwable t
+ ;; because the interrupted exception can be wrapped in a runtimeexception
+ (when-not (exception-cause? InterruptedException t)
+ (kill-fn t)
+ (reset! active false)
+ (throw t)))))
+ (.release notifier)))]
(.setDaemon timer-thread true)
(.setPriority timer-thread Thread/MAX_PRIORITY)
(.start timer-thread)
@@ -73,32 +71,36 @@
:lock lock
:cancel-notifier notifier}))
-(defn- check-active! [timer]
+(defn- check-active!
+ [timer]
(when-not @(:active timer)
(throw (IllegalStateException. "Timer is not active"))))
-(defnk schedule [timer delay-secs afn :check-active true]
+(defnk schedule
+ [timer delay-secs afn :check-active true]
(when check-active (check-active! timer))
(let [id (uuid)
^PriorityQueue queue (:queue timer)]
(locking (:lock timer)
- (.add queue [(+ (current-time-millis) (secs-to-millis-long delay-secs)) afn id])
- )))
+ (.add queue [(+ (current-time-millis) (secs-to-millis-long delay-secs)) afn id]))))
-(defn schedule-recurring [timer delay-secs recur-secs afn]
+(defn schedule-recurring
+ [timer delay-secs recur-secs afn]
(schedule timer
delay-secs
(fn this []
(afn)
- (schedule timer recur-secs this :check-active false)) ; this avoids a race condition with cancel-timer
- ))
+ ; this avoids a race condition with cancel-timer
+ (schedule timer recur-secs this :check-active false))))
-(defn cancel-timer [timer]
+(defn cancel-timer
+ [timer]
(check-active! timer)
(locking (:lock timer)
(reset! (:active timer) false)
(.interrupt (:timer-thread timer)))
(.acquire (:cancel-notifier timer)))
-(defn timer-waiting? [timer]
+(defn timer-waiting?
+ [timer]
(Time/isThreadWaiting (:timer-thread timer)))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2278fc96/storm-core/src/clj/backtype/storm/tuple.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/tuple.clj b/storm-core/src/clj/backtype/storm/tuple.clj
index eaec818..29df727 100644
--- a/storm-core/src/clj/backtype/storm/tuple.clj
+++ b/storm-core/src/clj/backtype/storm/tuple.clj
@@ -13,11 +13,12 @@
;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;; See the License for the specific language governing permissions and
;; limitations under the License.
+
(ns backtype.storm.tuple
- (:use [backtype.storm bootstrap])
- )
+ (:use [backtype.storm bootstrap]))
(bootstrap)
-(defn list-hash-code [^List alist]
+(defn list-hash-code
+ [^List alist]
(.hashCode alist))