You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2016/12/03 15:50:01 UTC
[01/10] storm git commit: STORM-1281: LocalCluster,
testing4j and testing.clj to java
Repository: storm
Updated Branches:
refs/heads/master 612aea66e -> 607ba8a9e
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/scheduler/multitenant_scheduler_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/scheduler/multitenant_scheduler_test.clj b/storm-core/test/clj/org/apache/storm/scheduler/multitenant_scheduler_test.clj
index c7b5050..1c3d94a 100644
--- a/storm-core/test/clj/org/apache/storm/scheduler/multitenant_scheduler_test.clj
+++ b/storm-core/test/clj/org/apache/storm/scheduler/multitenant_scheduler_test.clj
@@ -15,7 +15,7 @@
;; limitations under the License.
(ns org.apache.storm.scheduler.multitenant-scheduler-test
(:use [clojure test])
- (:use [org.apache.storm config testing log])
+ (:use [org.apache.storm config log])
(:import [org.apache.storm.generated StormTopology])
(:import [org.apache.storm.daemon.nimbus Nimbus$StandaloneINimbus])
(:import [org.apache.storm.scheduler Cluster SupervisorDetails WorkerSlot ExecutorDetails
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/scheduler_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/scheduler_test.clj b/storm-core/test/clj/org/apache/storm/scheduler_test.clj
index 003c650..1c479a7 100644
--- a/storm-core/test/clj/org/apache/storm/scheduler_test.clj
+++ b/storm-core/test/clj/org/apache/storm/scheduler_test.clj
@@ -15,7 +15,7 @@
;; limitations under the License.
(ns org.apache.storm.scheduler-test
(:use [clojure test])
- (:use [org.apache.storm util config testing])
+ (:use [org.apache.storm util config])
(:import [org.apache.storm.scheduler EvenScheduler])
(:import [org.apache.storm.daemon.nimbus Nimbus$StandaloneINimbus])
(:import [org.apache.storm.generated StormTopology])
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/security/auth/auth_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/security/auth/auth_test.clj b/storm-core/test/clj/org/apache/storm/security/auth/auth_test.clj
index 857ae6a..4ea6b45 100644
--- a/storm-core/test/clj/org/apache/storm/security/auth/auth_test.clj
+++ b/storm-core/test/clj/org/apache/storm/security/auth/auth_test.clj
@@ -31,17 +31,15 @@
(:import [java.security Principal AccessController])
(:import [javax.security.auth Subject])
(:import [java.net InetAddress])
- (:import [org.apache.storm Config])
+ (:import [org.apache.storm Config Testing Testing$Condition])
(:import [org.apache.storm.generated AuthorizationException])
(:import [org.apache.storm.daemon.nimbus Nimbus$StandaloneINimbus])
- (:import [org.apache.storm.utils NimbusClient ConfigUtils])
+ (:import [org.apache.storm.utils NimbusClient ConfigUtils Time])
(:import [org.apache.storm.security.auth.authorizer SimpleWhitelistAuthorizer SimpleACLAuthorizer])
(:import [org.apache.storm.security.auth AuthUtils ThriftServer ThriftClient ShellBasedGroupsMapping
ReqContext SimpleTransportPlugin KerberosPrincipalToLocal ThriftConnectionType])
(:import [org.apache.storm.daemon StormCommon])
(:use [org.apache.storm util config])
- (:use [org.apache.storm.daemon common])
- (:use [org.apache.storm testing])
(:import [org.apache.storm.generated Nimbus Nimbus$Client Nimbus$Iface StormTopology SubmitOptions
KillOptions RebalanceOptions ClusterSummary TopologyInfo Nimbus$Processor]
(org.json.simple JSONValue))
@@ -137,7 +135,7 @@
ThriftConnectionType/NIMBUS)]
(.addShutdownHook (Runtime/getRuntime) (Thread. (fn [] (.stop server))))
(.start (Thread. #(.serve server)))
- (wait-for-condition #(.isServing server))
+ (Testing/whileTimeout (reify Testing$Condition (exec [this] (not (.isServing server)))) (fn [] (Time/sleep 100)))
server ))
(defmacro with-server [args & body]
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj b/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
index af492c6..0c2ee2e 100644
--- a/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
+++ b/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
@@ -21,17 +21,23 @@
DistributedRPCInvocations$Processor]
[org.apache.storm.daemon DrpcServer])
(:import [org.apache.storm Config])
+ (:import [org.apache.storm Testing Testing$Condition])
(:import [org.apache.storm.security.auth ReqContext SingleUserPrincipal ThriftServer ThriftConnectionType])
- (:import [org.apache.storm.utils DRPCClient ConfigUtils])
+ (:import [org.apache.storm.utils DRPCClient ConfigUtils Time])
(:import [org.apache.storm.drpc DRPCInvocationsClient])
(:import [java.util.concurrent TimeUnit])
(:import [javax.security.auth Subject])
(:use [org.apache.storm util config log])
- (:use [org.apache.storm.daemon common])
- (:use [org.apache.storm testing])
(:import [org.apache.storm.utils Utils]))
-(def DRPC-TIMEOUT-SEC (* (/ TEST-TIMEOUT-MS 1000) 2))
+(defmacro with-timeout
+ [millis unit & body]
+ `(let [f# (future ~@body)]
+ (try
+ (.get f# ~millis ~unit)
+ (finally (future-cancel f#)))))
+
+(def DRPC-TIMEOUT-SEC (* (/ Testing/TEST_TIMEOUT_MS 1000) 2))
(defn launch-server [conf drpcAznClass transportPluginClass login-cfg client-port invocations-port]
(let [conf (if drpcAznClass (assoc conf DRPC-AUTHORIZER drpcAznClass) conf)
@@ -50,10 +56,10 @@
(log-message "storm conf:" conf)
(log-message "Starting DRPC invocation server ... " invocations-port)
(.start (Thread. #(.serve invoke-server)))
- (wait-for-condition #(.isServing invoke-server))
+ (Testing/whileTimeout (reify Testing$Condition (exec [this] (not (.isServing invoke-server)))) (fn [] (Time/sleep 100)))
(log-message "Starting DRPC handler server ... " client-port)
(.start (Thread. #(.serve handler-server)))
- (wait-for-condition #(.isServing handler-server))
+ (Testing/whileTimeout (reify Testing$Condition (exec [this] (not (.isServing handler-server)))) (fn [] (Time/sleep 100)))
[handler-server invoke-server]))
(defmacro with-server [args & body]
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/security/auth/nimbus_auth_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/security/auth/nimbus_auth_test.clj b/storm-core/test/clj/org/apache/storm/security/auth/nimbus_auth_test.clj
index 2117b07..edd8f99 100644
--- a/storm-core/test/clj/org/apache/storm/security/auth/nimbus_auth_test.clj
+++ b/storm-core/test/clj/org/apache/storm/security/auth/nimbus_auth_test.clj
@@ -15,11 +15,10 @@
;; limitations under the License.
(ns org.apache.storm.security.auth.nimbus-auth-test
(:use [clojure test])
- (:require [org.apache.storm [testing :as testing]])
(:require [org.apache.storm.security.auth [auth-test :refer [nimbus-timeout]]])
(:import [java.nio ByteBuffer])
(:import [java.util Optional])
- (:import [org.apache.storm Config])
+ (:import [org.apache.storm Config LocalCluster$Builder])
(:import [org.apache.storm.blobstore BlobStore])
(:import [org.apache.storm.utils NimbusClient ConfigUtils])
(:import [org.apache.storm.generated NotAliveException StormBase])
@@ -30,32 +29,25 @@
(:import [org.apache.storm.utils Utils])
(:import [org.apache.storm.cluster IStormClusterState])
(:import [org.mockito Mockito Matchers])
- (:use [org.apache.storm util config log testing])
+ (:use [org.apache.storm util config log])
(:require [conjure.core])
(:use [conjure core]))
-(defn launch-test-cluster [nimbus-port login-cfg aznClass transportPluginClass]
- (let [conf {NIMBUS-AUTHORIZER aznClass
+(defn to-conf [nimbus-port login-cfg aznClass transportPluginClass]
+ (let [conf {NIMBUS-AUTHORIZER aznClass
NIMBUS-THRIFT-PORT nimbus-port
STORM-THRIFT-TRANSPORT-PLUGIN transportPluginClass }
- conf (if login-cfg (merge conf {"java.security.auth.login.config" login-cfg}) conf)
- cluster-map (testing/mk-local-storm-cluster :supervisors 0
- :ports-per-supervisor 0
- :daemon-conf conf)
- nimbus-server (ThriftServer. (:daemon-conf cluster-map)
- (Nimbus$Processor. (:nimbus cluster-map))
- ThriftConnectionType/NIMBUS)]
- (.addShutdownHook (Runtime/getRuntime) (Thread. (fn [] (.stop nimbus-server))))
- (.start (Thread. #(.serve nimbus-server)))
- (testing/wait-for-condition #(.isServing nimbus-server))
- [cluster-map nimbus-server]))
+ conf (if login-cfg (merge conf {"java.security.auth.login.config" login-cfg}) conf)]
+ conf))
(defmacro with-test-cluster [args & body]
- `(let [[cluster-map# nimbus-server#] (launch-test-cluster ~@args)]
- ~@body
- (log-debug "Shutdown cluster from macro")
- (testing/kill-local-storm-cluster cluster-map#)
- (.stop nimbus-server#)))
+ `(let [conf# (to-conf ~@args)]
+ (with-open [_# (.build (doto (LocalCluster$Builder. )
+ (.withNimbusDaemon)
+ (.withDaemonConf conf#)
+ (.withSupervisors 0)
+ (.withPortsPerSupervisor 0)))]
+ ~@body)))
(deftest Simple-authentication-test
(let [port (Utils/getAvailablePort)]
@@ -76,11 +68,15 @@
blob-store (Mockito/mock BlobStore)
topo-name "topo-name"]
(.thenReturn (Mockito/when (.getTopoId cluster-state topo-name)) (Optional/empty))
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store
- :nimbus-daemon true
- :daemon-conf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"
- NIMBUS-THRIFT-PORT port
- STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.SimpleTransportPlugin"}]
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder.)
+ (.withClusterState cluster-state)
+ (.withBlobStore blob-store)
+ (.withNimbusDaemon)
+ (.withDaemonConf
+ {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"
+ NIMBUS-THRIFT-PORT port
+ STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.SimpleTransportPlugin"})))]
(let [storm-conf (merge (clojurify-structure (ConfigUtils/readStormConfig))
{STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.SimpleTransportPlugin"
STORM-NIMBUS-RETRY-TIMES 0})
@@ -99,11 +95,15 @@
topo-id "topo-name-1"]
(.thenReturn (Mockito/when (.getTopoId cluster-state topo-name)) (Optional/of topo-id))
(.thenReturn (Mockito/when (.readTopologyConf blob-store (Mockito/any String) (Mockito/anyObject))) {})
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store
- :nimbus-daemon true
- :daemon-conf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.DenyAuthorizer"
- NIMBUS-THRIFT-PORT port
- STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.SimpleTransportPlugin"}]
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder.)
+ (.withClusterState cluster-state)
+ (.withBlobStore blob-store)
+ (.withNimbusDaemon)
+ (.withDaemonConf
+ {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.DenyAuthorizer"
+ NIMBUS-THRIFT-PORT port
+ STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.SimpleTransportPlugin"})))]
(let [storm-conf (merge (clojurify-structure (ConfigUtils/readStormConfig))
{STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.SimpleTransportPlugin"
Config/NIMBUS_THRIFT_PORT port
@@ -159,12 +159,16 @@
topo-id "topo-name-1"]
(.thenReturn (Mockito/when (.getTopoId cluster-state topo-name)) (Optional/of topo-id))
(.thenReturn (Mockito/when (.readTopologyConf blob-store (Mockito/any String) (Mockito/anyObject))) {})
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store
- :nimbus-daemon true
- :daemon-conf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.DenyAuthorizer"
- "java.security.auth.login.config" "test/clj/org/apache/storm/security/auth/jaas_digest.conf"
- NIMBUS-THRIFT-PORT port
- STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.digest.DigestSaslTransportPlugin"}]
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder.)
+ (.withClusterState cluster-state)
+ (.withBlobStore blob-store)
+ (.withNimbusDaemon)
+ (.withDaemonConf
+ {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.DenyAuthorizer"
+ NIMBUS-THRIFT-PORT port
+ "java.security.auth.login.config" "test/clj/org/apache/storm/security/auth/jaas_digest.conf"
+ STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.digest.DigestSaslTransportPlugin"})))]
(let [storm-conf (merge (clojurify-structure (ConfigUtils/readStormConfig))
{STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.digest.DigestSaslTransportPlugin"
"java.security.auth.login.config" "test/clj/org/apache/storm/security/auth/jaas_digest.conf"
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/submitter_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/submitter_test.clj b/storm-core/test/clj/org/apache/storm/submitter_test.clj
index 3d858e0..e4e9385 100644
--- a/storm-core/test/clj/org/apache/storm/submitter_test.clj
+++ b/storm-core/test/clj/org/apache/storm/submitter_test.clj
@@ -15,7 +15,7 @@
;; limitations under the License.
(ns org.apache.storm.submitter-test
(:use [clojure test])
- (:use [org.apache.storm config testing])
+ (:use [org.apache.storm config])
(:import [org.apache.storm StormSubmitter])
)
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/subtopology_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/subtopology_test.clj b/storm-core/test/clj/org/apache/storm/subtopology_test.clj
deleted file mode 100644
index 3c759b3..0000000
--- a/storm-core/test/clj/org/apache/storm/subtopology_test.clj
+++ /dev/null
@@ -1,62 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.subtopology-test
- (:use [clojure test])
- (:import [org.apache.storm.topology TopologyBuilder])
- (:import [org.apache.storm.testing TestWordSpout PrepareBatchBolt BatchRepeatA BatchProcessWord BatchNumberList])
- (:import [org.apache.storm.coordination BatchSubtopologyBuilder])
- (:use [org.apache.storm testing])
- (:use [org.apache.storm.daemon common]))
-
-;; todo: need to configure coordinatedbolt with streams that aren't subscribed to, should auto-anchor those to the final
-;; coordination tuple... find all streams that aren't subscribed to
-;; having trouble with this test, commenting for now
-;; (deftest test-batch-subtopology
-;; (with-local-cluster [cluster :supervisors 4]
-;; (letlocals
-;; (bind builder (TopologyBuilder.))
-;; (.setSpout builder "spout" (TestWordSpout.))
-;; (-> (.setBolt builder "identity" (PrepareBatchBolt. (Fields. ["id" "word"])) 3)
-;; (.shuffleGrouping "spout")
-;; )
-;; (bind batch-builder (BatchSubtopologyBuilder. "for-a" (BatchRepeatA.) 2))
-;; (-> (.getMasterDeclarer batch-builder)
-;; (.shuffleGrouping "identity"))
-;; (-> (.setBolt batch-builder "process" (BatchProcessWord.) 2)
-;; (.fieldsGrouping "for-a" "multi" (Fields. ["id"])))
-;; (-> (.setBolt batch-builder "joiner" (BatchNumberList. "for-a") 2)
-;; (.fieldsGrouping "process" (Fields. ["id"]))
-;; (.fieldsGrouping "for-a" "single" (Fields. ["id"]))
-;; )
-;;
-;; (.extendTopology batch-builder builder)
-;;
-;; (bind results (complete-topology cluster
-;; (.createTopology builder)
-;; :storm-conf {TOPOLOGY-DEBUG true}
-;; :mock-sources {"spout" [
-;; ["ccacccaa"]
-;; ["bbb"]
-;; ["ba"]
-;; ]}
-;; ))
-;; (is (ms= [
-;; ["ccacccaa" [2 6 7]]
-;; ["bbb" []]
-;; ["ba" [1]]
-;; ]
-;; (read-tuples results "joiner")))
-;; )))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/tick_tuple_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/tick_tuple_test.clj b/storm-core/test/clj/org/apache/storm/tick_tuple_test.clj
index 99ad957..f478dd4 100644
--- a/storm-core/test/clj/org/apache/storm/tick_tuple_test.clj
+++ b/storm-core/test/clj/org/apache/storm/tick_tuple_test.clj
@@ -15,10 +15,9 @@
;; limitations under the License.
(ns org.apache.storm.tick-tuple-test
(:use [clojure test])
- (:use [org.apache.storm testing config])
+ (:use [org.apache.storm config])
(:use [org.apache.storm.internal clojure])
- (:use [org.apache.storm.daemon common])
- (:import [org.apache.storm Thrift])
+ (:import [org.apache.storm Thrift LocalCluster$Builder])
(:import [org.apache.storm.utils Utils]))
(defbolt noop-bolt ["tuple"] {:prepare true}
@@ -32,7 +31,8 @@
(nextTuple [])))
(deftest test-tick-tuple-works-with-system-bolt
- (with-simulated-time-local-cluster [cluster]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)))]
(let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails noop-spout)}
{"2" (Thrift/prepareBoltDetails
@@ -40,11 +40,11 @@
(Thrift/prepareFieldsGrouping ["tuple"])}
noop-bolt)})]
(try
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"test"
{TOPOLOGY-TICK-TUPLE-FREQ-SECS 1}
topology)
- (advance-cluster-time cluster 2)
+ (.advanceClusterTime cluster 2)
;; if reaches here, it means everything works ok.
(is true)
(catch Exception e
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/transactional_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/transactional_test.clj b/storm-core/test/clj/org/apache/storm/transactional_test.clj
index b8af518..358a4ba 100644
--- a/storm-core/test/clj/org/apache/storm/transactional_test.clj
+++ b/storm-core/test/clj/org/apache/storm/transactional_test.clj
@@ -15,7 +15,7 @@
;; limitations under the License.
(ns org.apache.storm.transactional-test
(:use [clojure test])
- (:import [org.apache.storm Constants])
+ (:import [org.apache.storm Constants LocalCluster$Builder Testing])
(:import [org.apache.storm.topology TopologyBuilder])
(:import [org.apache.storm.transactional TransactionalSpoutCoordinator ITransactionalSpout ITransactionalSpout$Coordinator TransactionAttempt
TransactionalTopologyBuilder])
@@ -26,7 +26,7 @@
(:import [org.apache.storm.coordination BatchBoltExecutor])
(:import [org.apache.storm.utils RegisteredGlobalState])
(:import [org.apache.storm.tuple Fields])
- (:import [org.apache.storm.testing CountingBatchBolt MemoryTransactionalSpout
+ (:import [org.apache.storm.testing InProcessZookeeper CountingBatchBolt MemoryTransactionalSpout
KeyedCountingBatchBolt KeyedCountingCommitterBolt KeyedSummingBatchBolt
IdentityBolt CountingCommitBolt OpaqueMemoryTransactionalSpout])
(:import [org.apache.storm.utils ZookeeperAuthInfo Utils])
@@ -36,9 +36,8 @@
(:import [org.mockito Matchers Mockito])
(:import [org.mockito.exceptions.base MockitoAssertionError])
(:import [java.util HashMap Collections ArrayList])
- (:use [org.apache.storm testing util config])
- (:use [org.apache.storm.internal clojure])
- (:use [org.apache.storm.daemon common]))
+ (:use [org.apache.storm util config log])
+ (:use [org.apache.storm.internal clojure]))
;; Testing TODO:
;; * Test that it repeats the meta for a partitioned state (test partitioned emitter on its own)
@@ -116,7 +115,7 @@
(deftest test-coordinator
(let [coordinator-state (atom nil)
emit-capture (atom nil)]
- (with-inprocess-zookeeper zk-port
+ (with-open [zk (InProcessZookeeper. )]
(letlocals
(bind coordinator
(mk-coordinator-state-changer coordinator-state))
@@ -124,7 +123,7 @@
(merge (clojurify-structure (Utils/readDefaultConfig))
{TOPOLOGY-MAX-SPOUT-PENDING 4
TOPOLOGY-TRANSACTIONAL-ID "abc"
- STORM-ZOOKEEPER-PORT zk-port
+ STORM-ZOOKEEPER-PORT (.getPort zk)
STORM-ZOOKEEPER-SERVERS ["localhost"]
})
nil
@@ -238,11 +237,11 @@
;; test that transactions are independent
- (.execute bolt (test-tuple [attempt1-1]))
- (.execute bolt (test-tuple [attempt1-1]))
- (.execute bolt (test-tuple [attempt1-2]))
- (.execute bolt (test-tuple [attempt2-1]))
- (.execute bolt (test-tuple [attempt1-1]))
+ (.execute bolt (Testing/testTuple [attempt1-1]))
+ (.execute bolt (Testing/testTuple [attempt1-1]))
+ (.execute bolt (Testing/testTuple [attempt1-2]))
+ (.execute bolt (Testing/testTuple [attempt2-1]))
+ (.execute bolt (Testing/testTuple [attempt1-1]))
(finish! bolt attempt1-1)
@@ -251,7 +250,7 @@
"default" [[attempt1-1 3]]}
capture-atom)
- (.execute bolt (test-tuple [attempt1-2]))
+ (.execute bolt (Testing/testTuple [attempt1-2]))
(finish! bolt attempt2-1)
(verify-bolt-and-reset! {:ack [[attempt1-2]]
"default" [[attempt2-1 1]]}
@@ -283,9 +282,9 @@
(deftest test-rotating-transactional-state
;; test strict ordered vs not strict ordered
- (with-inprocess-zookeeper zk-port
+ (with-open [zk (InProcessZookeeper. )]
(let [conf (merge (clojurify-structure (Utils/readDefaultConfig))
- {STORM-ZOOKEEPER-PORT zk-port
+ {STORM-ZOOKEEPER-PORT (.getPort zk)
STORM-ZOOKEEPER-SERVERS ["localhost"]
})
state (TransactionalState/newUserState conf "id1" {})
@@ -331,12 +330,6 @@
(-> source (.get p) (.addAll data))
))
-(defn tracked-captured-topology [cluster topology]
- (let [{captured :capturer topology :topology} (capture-topology topology)
- tracked (mk-tracked-topology cluster topology)]
- (assoc tracked :capturer captured)
- ))
-
;; puts its collector and tuples into the global state to be used externally
(defbolt controller-bolt {} {:prepare true :params [state-id]}
[conf context collector]
@@ -365,7 +358,7 @@
(deftest test-transactional-topology
- (with-tracked-cluster [cluster]
+ (with-open [cluster (.build (.withSimulatedTime (.withTracked (LocalCluster$Builder. ))))]
(with-controller-bolt [controller collector tuples]
(letlocals
(bind data (mk-transactional-source))
@@ -424,14 +417,15 @@
["mango" 2]
["zebra" 1]]})
- (bind topo-info (tracked-captured-topology
+ (bind topo-info (Testing/trackAndCaptureTopology
cluster
(.createTopology builder)))
- (submit-local-topology (:nimbus cluster)
- "transactional-test"
- {TOPOLOGY-MAX-SPOUT-PENDING 2}
- (:topology topo-info))
- (advance-cluster-time cluster 11)
+ (log-message "TOPO INFO \n" (pr-str topo-info))
+ (.submitTopology cluster
+ "transactional-test"
+ {TOPOLOGY-MAX-SPOUT-PENDING 2}
+ (.topology topo-info))
+ (.advanceClusterTime cluster 11)
(bind ack-tx! (fn [txid]
(let [[to-ack not-to-ack] (separate
#(-> %
@@ -456,42 +450,45 @@
;; only check default streams
(bind verify! (fn [expected]
- (let [results (-> topo-info :capturer .getResults)]
+ (let [results (-> topo-info .capturer .getResults)]
(doseq [[component tuples] expected
- :let [emitted (->> (read-tuples results
+ :let [emitted (->> (Testing/readTuples results
component
"default")
(map normalize-tx-tuple))]]
- (is (ms= tuples emitted)))
+ (log-message "\t\t!!!!! " component)
+ (is (Testing/multiseteq tuples emitted)))
(.clear results)
)))
- (tracked-wait topo-info 2)
- (verify! {"sum" [[1 "dog" 3]
- [1 "cat" 5]
- [1 "mango" 6]
- [1 "happy" 11]
- [2 "apple" 1]
- [2 "dog" 3]
- [2 "zebra" 1]]
+ (Testing/trackedWait topo-info (int 2))
+ (log-message "\n\n\t\t!!!!!!!!!!!! 1\n\n")
+ (verify! {"sum" [[(int 1) "dog" 3]
+ [(int 1) "cat" 5]
+ [(int 1) "mango" 6]
+ [(int 1) "happy" 11]
+ [(int 2) "apple" 1]
+ [(int 2) "dog" 3]
+ [(int 2) "zebra" 1]]
"count" []
"count2" []
- "global" [[1 6]
- [2 3]]
+ "global" [[(int 1) (int 6)]
+ [(int 2) (int 3)]]
"gcommit" []})
(ack-tx! 1)
- (tracked-wait topo-info 1)
+ (Testing/trackedWait topo-info (int 1))
+ (log-message "\n\n\t\t!!!!!!!!!!!! 2\n\n")
(verify! {"sum" []
- "count" [[1 "dog" 1]
- [1 "cat" 2]
- [1 "mango" 2]
- [1 "happy" 1]]
- "count2" [[1 "dog" 2]
- [1 "cat" 2]
- [1 "mango" 2]
- [1 "happy" 2]]
+ "count" [[(int 1) "dog" (int 1)]
+ [(int 1) "cat" (int 2)]
+ [(int 1) "mango" (int 2)]
+ [(int 1) "happy" (int 1)]]
+ "count2" [[(int 1) "dog" (int 2)]
+ [(int 1) "cat" (int 2)]
+ [(int 1) "mango" (int 2)]
+ [(int 1) "happy" (int 2)]]
"global" []
- "gcommit" [[1 6]]})
+ "gcommit" [[(int 1) (int 6)]]})
(add-transactional-data data
{0 [["a" 1]
@@ -505,151 +502,106 @@
3 [["a" 2]]})
(ack-tx! 1)
- (tracked-wait topo-info 1)
- (verify! {"sum" [[3 "a" 5]
- [3 "b" 2]
- [3 "d" 4]
- [3 "c" 1]
- [3 "e" 7]]
+ (Testing/trackedWait topo-info (int 1))
+ (log-message "\n\n\t\t!!!!!!!!!!!! 3\n\n")
+ (verify! {"sum" [[(int 3) "a" 5]
+ [(int 3) "b" 2]
+ [(int 3) "d" 4]
+ [(int 3) "c" 1]
+ [(int 3) "e" 7]]
"count" []
"count2" []
- "global" [[3 7]]
+ "global" [[(int 3) (int 7)]]
"gcommit" []})
(ack-tx! 3)
(ack-tx! 2)
- (tracked-wait topo-info 1)
+ (Testing/trackedWait topo-info (int 1))
+ (log-message "\n\n\t\t!!!!!!!!!!!! 4\n\n")
(verify! {"sum" []
- "count" [[2 "apple" 1]
- [2 "dog" 1]
- [2 "zebra" 1]]
- "count2" [[2 "apple" 2]
- [2 "dog" 2]
- [2 "zebra" 2]]
+ "count" [[(int 2) "apple" (int 1)]
+ [(int 2) "dog" (int 1)]
+ [(int 2) "zebra" (int 1)]]
+ "count2" [[(int 2) "apple" (int 2)]
+ [(int 2) "dog" (int 2)]
+ [(int 2) "zebra" (int 2)]]
"global" []
- "gcommit" [[2 3]]})
+ "gcommit" [[(int 2) (int 3)]]})
(fail-tx! 2)
- (tracked-wait topo-info 2)
-
- (verify! {"sum" [[2 "apple" 1]
- [2 "dog" 3]
- [2 "zebra" 1]
- [3 "a" 5]
- [3 "b" 2]
- [3 "d" 4]
- [3 "c" 1]
- [3 "e" 7]]
+ (Testing/trackedWait topo-info (int 2))
+
+ (log-message "\n\n\t\t!!!!!!!!!!!! 5\n\n")
+ (verify! {"sum" [[(int 2) "apple" 1]
+ [(int 2) "dog" 3]
+ [(int 2) "zebra" 1]
+ [(int 3) "a" 5]
+ [(int 3) "b" 2]
+ [(int 3) "d" 4]
+ [(int 3) "c" 1]
+ [(int 3) "e" 7]]
"count" []
"count2" []
- "global" [[2 3]
- [3 7]]
+ "global" [[(int 2) (int 3)]
+ [(int 3) (int 7)]]
"gcommit" []})
(ack-tx! 2)
- (tracked-wait topo-info 1)
+ (Testing/trackedWait topo-info (int 1))
+ (log-message "\n\n\t\t!!!!!!!!!!!! 6\n\n")
(verify! {"sum" []
- "count" [[2 "apple" 1]
- [2 "dog" 1]
- [2 "zebra" 1]]
- "count2" [[2 "apple" 2]
- [2 "dog" 2]
- [2 "zebra" 2]]
+ "count" [[(int 2) "apple" (int 1)]
+ [(int 2) "dog" (int 1)]
+ [(int 2) "zebra" (int 1)]]
+ "count2" [[(int 2) "apple" (int 2)]
+ [(int 2) "dog" (int 2)]
+ [(int 2) "zebra" (int 2)]]
"global" []
- "gcommit" [[2 3]]})
+ "gcommit" [[(int 2) (int 3)]]})
(ack-tx! 2)
(ack-tx! 3)
- (tracked-wait topo-info 2)
- (verify! {"sum" [[4 "c" 14]]
- "count" [[3 "a" 3]
- [3 "b" 1]
- [3 "d" 1]
- [3 "c" 1]
- [3 "e" 1]]
- "count2" [[3 "a" 2]
- [3 "b" 2]
- [3 "d" 2]
- [3 "c" 2]
- [3 "e" 2]]
- "global" [[4 2]]
- "gcommit" [[3 7]]})
+ (Testing/trackedWait topo-info (int 2))
+ (log-message "\n\n\t\t!!!!!!!!!!!! 7\n\n")
+ (verify! {"sum" [[(int 4) "c" 14]]
+ "count" [[(int 3) "a" (int 3)]
+ [(int 3) "b" (int 1)]
+ [(int 3) "d" (int 1)]
+ [(int 3) "c" (int 1)]
+ [(int 3) "e" (int 1)]]
+ "count2" [[(int 3) "a" (int 2)]
+ [(int 3) "b" (int 2)]
+ [(int 3) "d" (int 2)]
+ [(int 3) "c" (int 2)]
+ [(int 3) "e" (int 2)]]
+ "global" [[(int 4) (int 2)]]
+ "gcommit" [[(int 3) (int 7)]]})
(ack-tx! 4)
(ack-tx! 3)
- (tracked-wait topo-info 2)
+ (Testing/trackedWait topo-info (int 2))
+ (log-message "\n\n\t\t!!!!!!!!!!!! 8\n\n")
(verify! {"sum" []
- "count" [[4 "c" 2]]
- "count2" [[4 "c" 2]]
- "global" [[5 0]]
- "gcommit" [[4 2]]})
+ "count" [[(int 4) "c" (int 2)]]
+ "count2" [[(int 4) "c" (int 2)]]
+ "global" [[(int 5) (int 0)]]
+ "gcommit" [[(int 4) (int 2)]]})
(ack-tx! 5)
(ack-tx! 4)
- (tracked-wait topo-info 2)
+ (Testing/trackedWait topo-info (int 2))
+ (log-message "\n\n\t\t!!!!!!!!!!!! 9\n\n")
(verify! {"sum" []
"count" []
"count2" []
- "global" [[6 0]]
- "gcommit" [[5 0]]})
+ "global" [[(int 6) (int 0)]]
+ "gcommit" [[(int 5) (int 0)]]})
- (-> topo-info :capturer .getAndClearResults)
+ (-> topo-info .capturer .getAndClearResults)
))))
-(deftest test-transactional-topology-restart
- (with-simulated-time-local-cluster [cluster]
- (letlocals
- (bind data (mk-transactional-source))
- (bind builder (TransactionalTopologyBuilder.
- "id"
- "spout"
- (MemoryTransactionalSpout. data
- (Fields. ["word"])
- 3)
- 2))
-
- (-> builder
- (.setBolt "count" (CountingCommitBolt.) 2)
- (.globalGrouping "spout"))
-
- (add-transactional-data data
- {0 [["a"]
- ["b"]
- ["c"]
- ["d"]]
- 1 [["d"]
- ["c"]]
- })
-
- (bind results (complete-topology cluster
- (.buildTopology builder)
- :storm-conf {TOPOLOGY-DEBUG true
- TOPOLOGY-MESSAGE-TIMEOUT-SECS 300} ;;simulated time can take a while for things to calm down
- :cleanup-state false))
-
- (is (ms= [[5] [0] [1] [0]] (->> (read-tuples results "count")
- (take 4)
- (map (partial drop 1))
- )))
-
- (add-transactional-data data
- {0 [["a"]
- ["b"]]
- })
-
- (bind results (complete-topology cluster (.buildTopology builder)
- :storm-conf {TOPOLOGY-DEBUG true
- TOPOLOGY-MESSAGE-TIMEOUT-SECS 300}))
-
- ;; need to do it this way (check for nothing transaction) because there is one transaction already saved up before that emits nothing (because of how memorytransctionalspout detects partition completion)
- (is (ms= [[0] [0] [2] [0]] (->> (read-tuples results "count")
- (take 4)
- (map (partial drop 1))
- )))
- )))
-
(deftest test-opaque-transactional-topology
- (with-tracked-cluster [cluster]
+ (with-open [cluster (.build (.withSimulatedTime (.withTracked (LocalCluster$Builder. ))))]
(with-controller-bolt [controller collector tuples]
(letlocals
(bind data (mk-transactional-source))
@@ -678,15 +630,14 @@
["apple"]
["dog"]]})
- (bind topo-info (tracked-captured-topology
+ (bind topo-info (Testing/trackAndCaptureTopology
cluster
(.createTopology builder)))
- (submit-local-topology (:nimbus cluster)
- "transactional-test"
- {TOPOLOGY-MAX-SPOUT-PENDING 2
- }
- (:topology topo-info))
- (advance-cluster-time cluster 11)
+ (.submitTopology cluster
+ "transactional-test"
+ {TOPOLOGY-MAX-SPOUT-PENDING 2}
+ (.topology topo-info))
+ (.advanceClusterTime cluster 11)
(bind ack-tx! (fn [txid]
(let [[to-ack not-to-ack] (separate
#(-> %
@@ -711,29 +662,29 @@
;; only check default streams
(bind verify! (fn [expected]
- (let [results (-> topo-info :capturer .getResults)]
+ (let [results (-> topo-info .capturer .getResults)]
(doseq [[component tuples] expected
- :let [emitted (->> (read-tuples results
+ :let [emitted (->> (Testing/readTuples results
component
"default")
(map normalize-tx-tuple))]]
- (is (ms= tuples emitted)))
+ (is (Testing/multiseteq tuples emitted)))
(.clear results)
)))
- (tracked-wait topo-info 2)
+ (Testing/trackedWait topo-info (int 2))
(verify! {"count" []})
(ack-tx! 1)
- (tracked-wait topo-info 1)
+ (Testing/trackedWait topo-info (int 1))
- (verify! {"count" [[1 "dog" 1]
- [1 "cat" 1]]})
+ (verify! {"count" [[(int 1) "dog" (int 1)]
+ [(int 1) "cat" (int 1)]]})
(ack-tx! 2)
(ack-tx! 1)
- (tracked-wait topo-info 2)
+ (Testing/trackedWait topo-info (int 2))
- (verify! {"count" [[2 "apple" 1]
- [2 "dog" 1]]})
+ (verify! {"count" [[(int 2) "apple" (int 1)]
+ [(int 2) "dog" (int 1)]]})
))))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/trident/state_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/trident/state_test.clj b/storm-core/test/clj/org/apache/storm/trident/state_test.clj
index 5f31175..4549d81 100644
--- a/storm-core/test/clj/org/apache/storm/trident/state_test.clj
+++ b/storm-core/test/clj/org/apache/storm/trident/state_test.clj
@@ -15,7 +15,6 @@
;; limitations under the License.
(ns org.apache.storm.trident.state-test
(:use [clojure test])
- (:require [org.apache.storm [testing :as t]])
(:import [org.apache.storm.trident.operation.builtin Count])
(:import [org.apache.storm.trident.state OpaqueValue])
(:import [org.apache.storm.utils Utils])
@@ -29,7 +28,6 @@
(:import [org.apache.zookeeper CreateMode ZooDefs ZooDefs$Ids])
(:import [org.mockito Matchers Mockito])
(:import [org.mockito.exceptions.base MockitoAssertionError])
- (:use [org.apache.storm.trident testing])
(:use [org.apache.storm config]))
(defn single-remove [map key]
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/trident/tuple_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/trident/tuple_test.clj b/storm-core/test/clj/org/apache/storm/trident/tuple_test.clj
index 0971afc..3876d49 100644
--- a/storm-core/test/clj/org/apache/storm/trident/tuple_test.clj
+++ b/storm-core/test/clj/org/apache/storm/trident/tuple_test.clj
@@ -15,11 +15,11 @@
;; limitations under the License.
(ns org.apache.storm.trident.tuple-test
(:use [clojure test])
- (:require [org.apache.storm [testing :as t]])
+ (:import [org.apache.storm.tuple Fields])
+ (:import [org.apache.storm Testing])
(:import [org.apache.storm.trident.tuple TridentTupleView TridentTupleView$ProjectionFactory
TridentTupleView$FreshOutputFactory TridentTupleView$OperationOutputFactory
- TridentTupleView$RootFactory])
- (:use [org.apache.storm.trident testing]))
+ TridentTupleView$RootFactory]))
(defmacro letlocals
[& body]
@@ -35,7 +35,7 @@
(deftest test-fresh
(letlocals
- (bind fresh-factory (TridentTupleView$FreshOutputFactory. (fields "a" "b" "c")))
+ (bind fresh-factory (TridentTupleView$FreshOutputFactory. (Fields. ["a" "b" "c"])))
(bind tt (.create fresh-factory [3 2 1]))
(is (= [3 2 1] tt))
(is (= 3 (.getValueByField tt "a")))
@@ -45,8 +45,8 @@
(deftest test-projection
(letlocals
- (bind fresh-factory (TridentTupleView$FreshOutputFactory. (fields "a" "b" "c" "d" "e")))
- (bind project-factory (TridentTupleView$ProjectionFactory. fresh-factory (fields "d" "a")))
+ (bind fresh-factory (TridentTupleView$FreshOutputFactory. (Fields. ["a" "b" "c" "d" "e"])))
+ (bind project-factory (TridentTupleView$ProjectionFactory. fresh-factory (Fields. ["d" "a"])))
(bind tt (.create fresh-factory [3 2 1 4 5]))
(bind tt2 (.create fresh-factory [9 8 7 6 10]))
@@ -63,9 +63,9 @@
(deftest test-appends
(letlocals
- (bind fresh-factory (TridentTupleView$FreshOutputFactory. (fields "a" "b" "c")))
- (bind append-factory (TridentTupleView$OperationOutputFactory. fresh-factory (fields "d" "e")))
- (bind append-factory2 (TridentTupleView$OperationOutputFactory. append-factory (fields "f")))
+ (bind fresh-factory (TridentTupleView$FreshOutputFactory. (Fields. ["a" "b" "c"])))
+ (bind append-factory (TridentTupleView$OperationOutputFactory. fresh-factory (Fields. ["d" "e"])))
+ (bind append-factory2 (TridentTupleView$OperationOutputFactory. append-factory (Fields. ["f"])))
(bind tt (.create fresh-factory [1 2 3]))
(bind tt2 (.create append-factory tt [4 5]))
@@ -79,14 +79,14 @@
(deftest test-root
(letlocals
- (bind root-factory (TridentTupleView$RootFactory. (fields "a" "b")))
- (bind storm-tuple (t/test-tuple ["a" 1]))
+ (bind root-factory (TridentTupleView$RootFactory. (Fields. ["a" "b"])))
+ (bind storm-tuple (Testing/testTuple ["a" 1]))
(bind tt (.create root-factory storm-tuple))
(is (= ["a" 1] tt))
(is (= "a" (.getValueByField tt "a")))
(is (= 1 (.getValueByField tt "b")))
- (bind append-factory (TridentTupleView$OperationOutputFactory. root-factory (fields "c")))
+ (bind append-factory (TridentTupleView$OperationOutputFactory. root-factory (Fields. ["c"])))
(bind tt2 (.create append-factory tt [3]))
(is (= ["a" 1 3] tt2))
@@ -97,11 +97,11 @@
(deftest test-complex
(letlocals
- (bind fresh-factory (TridentTupleView$FreshOutputFactory. (fields "a" "b" "c")))
- (bind append-factory1 (TridentTupleView$OperationOutputFactory. fresh-factory (fields "d")))
- (bind append-factory2 (TridentTupleView$OperationOutputFactory. append-factory1 (fields "e" "f")))
- (bind project-factory1 (TridentTupleView$ProjectionFactory. append-factory2 (fields "a" "f" "b")))
- (bind append-factory3 (TridentTupleView$OperationOutputFactory. project-factory1 (fields "c")))
+ (bind fresh-factory (TridentTupleView$FreshOutputFactory. (Fields. ["a" "b" "c"])))
+ (bind append-factory1 (TridentTupleView$OperationOutputFactory. fresh-factory (Fields. ["d"])))
+ (bind append-factory2 (TridentTupleView$OperationOutputFactory. append-factory1 (Fields. ["e" "f"])))
+ (bind project-factory1 (TridentTupleView$ProjectionFactory. append-factory2 (Fields. ["a" "f" "b"])))
+ (bind append-factory3 (TridentTupleView$OperationOutputFactory. project-factory1 (Fields. ["c"])))
(bind tt (.create fresh-factory [1 2 3]))
(bind tt2 (.create append-factory1 tt [4]))
@@ -123,11 +123,11 @@
(deftest test-ituple-interface
(letlocals
- (bind tt (TridentTupleView/createFreshTuple (fields "a" "b" "c") [1 2 3]))
+ (bind tt (TridentTupleView/createFreshTuple (Fields. ["a" "b" "c"]) [1 2 3]))
(is (= [1 2 3] tt))
(is (= ["a" "b" "c"] (.toList (.getFields tt))))
(is (true? (.contains tt "a")))
(is (false? (.contains tt "abcd")))
(is (= 0 (.fieldIndex tt "a")))
- (is (= [3 1] (.select tt (fields "c" "a"))))
+ (is (= [3 1] (.select tt (Fields. ["c" "a"]))))
))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/tuple_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/tuple_test.clj b/storm-core/test/clj/org/apache/storm/tuple_test.clj
index a88933e..5d5dea7 100644
--- a/storm-core/test/clj/org/apache/storm/tuple_test.clj
+++ b/storm-core/test/clj/org/apache/storm/tuple_test.clj
@@ -15,11 +15,12 @@
;; limitations under the License.
(ns org.apache.storm.tuple-test
(:use [clojure test])
- (:import [org.apache.storm.tuple Tuple])
- (:use [org.apache.storm testing]))
+ (:import [org.apache.storm Testing])
+ (:import [org.apache.storm.testing MkTupleParam])
+ (:import [org.apache.storm.tuple Tuple]))
(deftest test-lookup
- (let [ tuple (test-tuple [12 "hello"] :fields ["foo" "bar"]) ]
+ (let [ tuple (Testing/testTuple [12 "hello"] (doto (MkTupleParam. ) (.setFieldsList ["foo" "bar"]))) ]
(is (= 12 (tuple "foo")))
(is (= 12 (tuple :foo)))
(is (= 12 (:foo tuple)))
@@ -29,16 +30,16 @@
(is (= :notfound (tuple "404" :notfound)))))
(deftest test-indexed
- (let [ tuple (test-tuple [12 "hello"] :fields ["foo" "bar"]) ]
+ (let [ tuple (Testing/testTuple [12 "hello"] (doto (MkTupleParam. ) (.setFieldsList ["foo" "bar"]))) ]
(is (= 12 (nth tuple 0)))
(is (= "hello" (nth tuple 1)))))
(deftest test-seq
- (let [ tuple (test-tuple [12 "hello"] :fields ["foo" "bar"]) ]
+ (let [ tuple (Testing/testTuple [12 "hello"] (doto (MkTupleParam. ) (.setFieldsList ["foo" "bar"]))) ]
(is (= [["foo" 12] ["bar" "hello"]] (seq tuple)))))
(deftest test-map
- (let [tuple (test-tuple [12 "hello"] :fields ["foo" "bar"]) ]
+ (let [tuple (Testing/testTuple [12 "hello"] (doto (MkTupleParam. ) (.setFieldsList ["foo" "bar"]))) ]
(is (= {"foo" 42 "bar" "hello"} (.getMap (assoc tuple "foo" 42))))
(is (= {"foo" 42 "bar" "hello"} (.getMap (assoc tuple :foo 42))))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/versioned_store_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/versioned_store_test.clj b/storm-core/test/clj/org/apache/storm/versioned_store_test.clj
index 2ee7a27..5e71beb 100644
--- a/storm-core/test/clj/org/apache/storm/versioned_store_test.clj
+++ b/storm-core/test/clj/org/apache/storm/versioned_store_test.clj
@@ -15,15 +15,15 @@
;; limitations under the License.
(ns org.apache.storm.versioned-store-test
(:use [clojure test])
- (:use [org.apache.storm testing])
(:import [java.io File])
+ (:import [org.apache.storm.testing TmpPath])
(:import [org.apache.commons.io FileUtils])
(:import [org.apache.storm.utils VersionedStore]))
(defmacro defvstest [name [vs-sym] & body]
`(deftest ~name
- (with-local-tmp [dir#]
- (let [~vs-sym (VersionedStore. dir#)]
+ (with-open [dir# (TmpPath.)]
+ (let [~vs-sym (VersionedStore. (.getPath dir#))]
~@body
))))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
index 24ccda5..eb25566 100644
--- a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
@@ -43,6 +43,7 @@ import org.apache.storm.localizer.ILocalizer;
import org.apache.storm.scheduler.ISupervisor;
import org.apache.storm.utils.LocalState;
import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
import org.junit.Test;
public class SlotTest {
@@ -113,8 +114,7 @@ public class SlotTest {
@Test
public void testEmptyToEmpty() throws Exception {
- Time.startSimulatingAutoAdvanceOnSleep(1010);
- try {
+ try (SimulatedTime t = new SimulatedTime(1010)){
ILocalizer localizer = mock(ILocalizer.class);
LocalState state = mock(LocalState.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
@@ -125,15 +125,12 @@ public class SlotTest {
DynamicState nextState = Slot.handleEmpty(dynamicState, staticState);
assertEquals(MachineState.EMPTY, nextState.state);
assertTrue(Time.currentTimeMillis() > 1000);
- } finally {
- Time.stopSimulating();
}
}
@Test
public void testLaunchContainerFromEmpty() throws Exception {
- Time.startSimulatingAutoAdvanceOnSleep(1010);
- try {
+ try (SimulatedTime t = new SimulatedTime(1010)){
int port = 8080;
String topoId = "NEW";
List<ExecutorInfo> execList = mkExecutorInfoList(1,2,3,4,5);
@@ -210,16 +207,13 @@ public class SlotTest {
assertSame(newAssignment, nextState.currentAssignment);
assertSame(container, nextState.container);
assertTrue(Time.currentTimeMillis() > 2000);
- } finally {
- Time.stopSimulating();
}
}
@Test
public void testRelaunch() throws Exception {
- Time.startSimulatingAutoAdvanceOnSleep(1010);
- try {
+ try (SimulatedTime t = new SimulatedTime(1010)){
int port = 8080;
String topoId = "CURRENT";
List<ExecutorInfo> execList = mkExecutorInfoList(1,2,3,4,5);
@@ -260,15 +254,12 @@ public class SlotTest {
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
- } finally {
- Time.stopSimulating();
}
}
@Test
public void testReschedule() throws Exception {
- Time.startSimulatingAutoAdvanceOnSleep(1010);
- try {
+ try (SimulatedTime t = new SimulatedTime(1010)){
int port = 8080;
String cTopoId = "CURRENT";
List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
@@ -368,16 +359,13 @@ public class SlotTest {
assertSame(nAssignment, nextState.currentAssignment);
assertSame(nContainer, nextState.container);
assertTrue(Time.currentTimeMillis() > 4000);
- } finally {
- Time.stopSimulating();
}
}
@Test
public void testRunningToEmpty() throws Exception {
- Time.startSimulatingAutoAdvanceOnSleep(1010);
- try {
+ try (SimulatedTime t = new SimulatedTime(1010)){
int port = 8080;
String cTopoId = "CURRENT";
List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
@@ -432,15 +420,12 @@ public class SlotTest {
assertEquals(null, nextState.container);
assertEquals(null, nextState.currentAssignment);
assertTrue(Time.currentTimeMillis() > 3000);
- } finally {
- Time.stopSimulating();
}
}
@Test
public void testRunWithProfileActions() throws Exception {
- Time.startSimulatingAutoAdvanceOnSleep(1010);
- try {
+ try (SimulatedTime t = new SimulatedTime(1010)){
int port = 8080;
String cTopoId = "CURRENT";
List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
@@ -508,8 +493,6 @@ public class SlotTest {
assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.pendingStopProfileActions);
assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.profileActions);
assertTrue(Time.currentTimeMillis() > 5000);
- } finally {
- Time.stopSimulating();
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/jvm/org/apache/storm/daemon/worker/LogConfigManagerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/worker/LogConfigManagerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/worker/LogConfigManagerTest.java
index 3f9b2be..95491a4 100644
--- a/storm-core/test/jvm/org/apache/storm/daemon/worker/LogConfigManagerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/daemon/worker/LogConfigManagerTest.java
@@ -29,6 +29,7 @@ import org.apache.storm.generated.LogConfig;
import org.apache.storm.generated.LogLevel;
import org.apache.storm.generated.LogLevelAction;
import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,8 +81,7 @@ public class LogConfigManagerTest {
@Test
public void testLogResetShouldNotTriggerForFutureTime() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
long theFuture = Time.currentTimeMillis() + 1000;
TreeMap<String, LogLevel> config = new TreeMap<>();
config.put("foo", ll(theFuture));
@@ -90,15 +90,12 @@ public class LogConfigManagerTest {
LogConfigManager underTest = new LogConfigManagerUnderTest(atomConf);
underTest.resetLogLevels();
assertNotNull(atomConf.get());
- } finally {
- Time.stopSimulating();
}
}
@Test
public void testLogResetTriggersForPastTime() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
long past = Time.currentTimeMillis() - 1000;
TreeMap<String, LogLevel> config = new TreeMap<>();
config.put("foo", ll("INFO", "WARN", past));
@@ -107,8 +104,6 @@ public class LogConfigManagerTest {
LogConfigManager underTest = new LogConfigManagerUnderTest(atomConf);
underTest.resetLogLevels();
assertEquals(new TreeMap<>(), atomConf.get());
- } finally {
- Time.stopSimulating();
}
}
@@ -125,8 +120,7 @@ public class LogConfigManagerTest {
@Test
public void testLogResetResetsRootLoggerIfSet() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
long past = Time.currentTimeMillis() - 1000;
TreeMap<String, LogLevel> config = new TreeMap<>();
config.put(LogManager.ROOT_LOGGER_NAME, ll("DEBUG", "WARN", past));
@@ -136,15 +130,12 @@ public class LogConfigManagerTest {
underTest.resetLogLevels();
assertEquals(new TreeMap<>(), atomConf.get());
verify(underTest).setLoggerLevel(anyObject(), eq(LogManager.ROOT_LOGGER_NAME), eq("WARN"));
- } finally {
- Time.stopSimulating();
}
}
@Test
public void testLogResetsNamedLoggersWithPastTimeout() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
long past = Time.currentTimeMillis() - 1000;
TreeMap<String, LogLevel> config = new TreeMap<>();
config.put("my_debug_logger", ll("DEBUG", "INFO", past));
@@ -158,15 +149,12 @@ public class LogConfigManagerTest {
verify(underTest).setLoggerLevel(anyObject(), eq("my_debug_logger"), eq("INFO"));
verify(underTest).setLoggerLevel(anyObject(), eq("my_info_logger"), eq("WARN"));
verify(underTest).setLoggerLevel(anyObject(), eq("my_error_logger"), eq("INFO"));
- } finally {
- Time.stopSimulating();
}
}
@Test
public void testProcessRootLogLevelToDebugSetsLoggerAndTimeout2() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
LogConfig mockConfig = new LogConfig();
AtomicReference<TreeMap<String, LogLevel>> mockConfigAtom = new AtomicReference<>(null);
@@ -186,15 +174,12 @@ public class LogConfigManagerTest {
// defaults to INFO level when the logger isn't found previously
assertEquals("INFO", rootResult.get_reset_log_level());
assertEquals(inThirtySeconds, rootResult.get_reset_log_level_timeout_epoch());
- } finally {
- Time.stopSimulating();
}
}
@Test
public void testProcessRootLogLevelToDebugSetsLoggerAndTimeout() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
LogConfig mockConfig = new LogConfig();
AtomicReference<TreeMap<String, LogLevel>> mockConfigAtom = new AtomicReference<>(null);
long inThirtySeconds = Time.currentTimeMillis() + 30_000;
@@ -212,8 +197,6 @@ public class LogConfigManagerTest {
verify(underTest).setLoggerLevel(anyObject(), eq("my_debug_logger"), eq("DEBUG"));
verify(underTest).setLoggerLevel(anyObject(), eq("my_info_logger"), eq("INFO"));
verify(underTest).setLoggerLevel(anyObject(), eq("my_error_logger"), eq("ERROR"));
- } finally {
- Time.stopSimulating();
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/jvm/org/apache/storm/executor/error/ReportErrorTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/executor/error/ReportErrorTest.java b/storm-core/test/jvm/org/apache/storm/executor/error/ReportErrorTest.java
index 98a43e4..eaf2617 100644
--- a/storm-core/test/jvm/org/apache/storm/executor/error/ReportErrorTest.java
+++ b/storm-core/test/jvm/org/apache/storm/executor/error/ReportErrorTest.java
@@ -28,6 +28,7 @@ import org.apache.storm.Config;
import org.apache.storm.cluster.IStormClusterState;
import org.apache.storm.task.WorkerTopologyContext;
import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
import org.junit.Test;
public class ReportErrorTest {
@@ -49,8 +50,7 @@ public class ReportErrorTest {
conf.put(Config.TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS, 10);
conf.put(Config.TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL, 4);
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
ReportError report = new ReportError(conf, state, topo, comp, context);
report.report(new RuntimeException("ERROR-1"));
assertEquals(1, errorCount.get());
@@ -69,8 +69,6 @@ public class ReportErrorTest {
Time.advanceTime(2000);
report.report(new RuntimeException("ERROR-7"));
assertEquals(5, errorCount.get());
- } finally {
- Time.stopSimulating();
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/jvm/org/apache/storm/utils/TimeTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/utils/TimeTest.java b/storm-core/test/jvm/org/apache/storm/utils/TimeTest.java
index f4b3977..dcd5f36 100644
--- a/storm-core/test/jvm/org/apache/storm/utils/TimeTest.java
+++ b/storm-core/test/jvm/org/apache/storm/utils/TimeTest.java
@@ -19,6 +19,7 @@
package org.apache.storm.utils;
import org.junit.Test;
+import org.apache.storm.utils.Time.SimulatedTime;
import org.junit.Assert;
public class TimeTest {
@@ -41,71 +42,53 @@ public class TimeTest {
@Test
public void isSimulatingReturnsTrueDuringSimulationTest() {
Assert.assertFalse(Time.isSimulating());
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
Assert.assertTrue(Time.isSimulating());
- } finally {
- Time.stopSimulating();
}
}
@Test
public void shouldNotAdvanceTimeTest() {
- Time.startSimulating();
- try{
+ try (SimulatedTime t = new SimulatedTime()){
long current = Time.currentTimeMillis();
Time.advanceTime(0);
Assert.assertEquals(Time.deltaMs(current), 0);
- } finally {
- Time.stopSimulating();
}
}
@Test
public void shouldAdvanceForwardTest() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
long current = Time.currentTimeMillis();
Time.advanceTime(1000);
Assert.assertEquals(Time.deltaMs(current), 1000);
Time.advanceTime(500);
Assert.assertEquals(Time.deltaMs(current), 1500);
- } finally {
- Time.stopSimulating();
}
}
@Test(expected=IllegalArgumentException.class)
public void shouldThrowIfAttemptToAdvanceBackwardsTest() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
Time.advanceTime(-1500);
- } finally {
- Time.stopSimulating();
}
}
@Test
public void deltaSecsConvertsToSecondsTest() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
int current = Time.currentTimeSecs();
Time.advanceTime(1000);
Assert.assertEquals(Time.deltaSecs(current), 1);
- } finally {
- Time.stopSimulating();
}
}
@Test
public void deltaSecsTruncatesFractionalSecondsTest() {
- Time.startSimulating();
- try {
+ try (SimulatedTime t = new SimulatedTime()){
int current = Time.currentTimeSecs();
Time.advanceTime(1500);
Assert.assertEquals(Time.deltaSecs(current), 1, 0);
- } finally {
- Time.stopSimulating();
}
}
[02/10] storm git commit: STORM-1281: LocalCluster,
testing4j and testing.clj to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/nimbus_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/nimbus_test.clj b/storm-core/test/clj/org/apache/storm/nimbus_test.clj
index cc760a4..bb949d5 100644
--- a/storm-core/test/clj/org/apache/storm/nimbus_test.clj
+++ b/storm-core/test/clj/org/apache/storm/nimbus_test.clj
@@ -16,16 +16,18 @@
(ns org.apache.storm.nimbus-test
(:use [clojure test])
(:require [org.apache.storm [util :as util]])
- (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount
+ (:import [java.util.function UnaryOperator])
+ (:import [org.apache.storm.testing InProcessZookeeper MockLeaderElector TestWordCounter TestWordSpout TestGlobalCount
TestAggregatesCounter TestPlannerSpout TestPlannerBolt]
[org.apache.storm.blobstore BlobStore]
[org.apache.storm.nimbus InMemoryTopologyActionNotifier]
[org.apache.storm.daemon.nimbus Nimbus Nimbus$StandaloneINimbus]
[org.apache.storm.generated GlobalStreamId TopologyStatus SupervisorInfo StormTopology StormBase]
- [org.apache.storm Thrift MockAutoCred]
+ [org.apache.storm LocalCluster LocalCluster$Builder Thrift MockAutoCred Testing Testing$Condition]
[org.apache.storm.stats BoltExecutorStats StatsUtil]
[org.apache.storm.security.auth IGroupMappingServiceProvider IAuthorizer])
(:import [org.apache.storm.testing.staticmocking MockedZookeeper])
+ (:import [org.apache.storm.testing TmpPath])
(:import [org.apache.storm.scheduler INimbus])
(:import [org.mockito Mockito Matchers])
(:import [org.mockito.exceptions.base MockitoAssertionError])
@@ -38,18 +40,22 @@
(:import [java.util HashMap HashSet Optional])
(:import [java.io File])
(:import [javax.security.auth Subject])
- (:import [org.apache.storm.utils Time Utils Utils$UptimeComputer ConfigUtils IPredicate StormCommonInstaller]
+ (:import [org.apache.storm.utils Time Time$SimulatedTime Utils Utils$UptimeComputer ConfigUtils IPredicate StormCommonInstaller]
[org.apache.storm.utils.staticmocking ConfigUtilsInstaller UtilsInstaller])
(:import [org.apache.storm.zookeeper Zookeeper])
(:import [org.apache.commons.io FileUtils])
(:import [org.json.simple JSONValue])
(:import [org.apache.storm.daemon StormCommon])
(:import [org.apache.storm.cluster IStormClusterState StormClusterStateImpl ClusterStateContext ClusterUtils])
- (:use [org.apache.storm testing util config log converter])
+ (:use [org.apache.storm util config log])
(:require [conjure.core])
(:use [conjure core]))
+(defn- mk-nimbus
+ [conf inimbus blob-store leader-elector group-mapper cluster-state]
+ (Nimbus. conf inimbus cluster-state nil blob-store leader-elector group-mapper))
+
(defn- from-json
[^String str]
(if str
@@ -58,28 +64,30 @@
nil))
(defn storm-component->task-info [cluster storm-name]
- (let [storm-id (StormCommon/getStormId (:storm-cluster-state cluster) storm-name)
- nimbus (:nimbus cluster)]
+ (let [storm-id (StormCommon/getStormId (.getClusterState cluster) storm-name)
+ nimbus (.getNimbus cluster)]
(-> (.getUserTopology nimbus storm-id)
(#(StormCommon/stormTaskInfo % (from-json (.getTopologyConf nimbus storm-id))))
(Utils/reverseMap)
clojurify-structure)))
(defn getCredentials [cluster storm-name]
- (let [storm-id (StormCommon/getStormId (:storm-cluster-state cluster) storm-name)]
- (clojurify-crdentials (.credentials (:storm-cluster-state cluster) storm-id nil))))
+ (let [storm-id (StormCommon/getStormId (.getClusterState cluster) storm-name)
+ creds (.credentials (.getClusterState cluster) storm-id nil)]
+ (if creds (into {} (.get_creds creds)))))
(defn storm-component->executor-info [cluster storm-name]
- (let [storm-id (StormCommon/getStormId (:storm-cluster-state cluster) storm-name)
- nimbus (:nimbus cluster)
+ (let [storm-id (StormCommon/getStormId (.getClusterState cluster) storm-name)
+ nimbus (.getNimbus cluster)
storm-conf (from-json (.getTopologyConf nimbus storm-id))
topology (.getUserTopology nimbus storm-id)
task->component (clojurify-structure (StormCommon/stormTaskInfo topology storm-conf))
- state (:storm-cluster-state cluster)
+ state (.getClusterState cluster)
get-component (comp task->component first)]
- (->> (clojurify-assignment (.assignmentInfo state storm-id nil))
- :executor->node+port
- keys
+ (->> (.assignmentInfo state storm-id nil)
+ .get_executor_node_port
+ .keySet
+ clojurify-structure
(map (fn [e] {e (get-component e)}))
(apply merge)
(Utils/reverseMap)
@@ -87,26 +95,25 @@
(defn storm-num-workers [state storm-name]
(let [storm-id (StormCommon/getStormId state storm-name)
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))]
- (count (clojurify-structure (Utils/reverseMap (:executor->node+port assignment))))
- ))
+ assignment (.assignmentInfo state storm-id nil)]
+ (.size (Utils/reverseMap (.get_executor_node_port assignment)))))
(defn topology-nodes [state storm-name]
(let [storm-id (StormCommon/getStormId state storm-name)
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))]
+ assignment (.assignmentInfo state storm-id nil)]
(->> assignment
- :executor->node+port
- vals
- (map first)
+ .get_executor_node_port
+ .values
+ (map (fn [np] (.get_node np)))
set
)))
(defn topology-slots [state storm-name]
(let [storm-id (StormCommon/getStormId state storm-name)
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))]
+ assignment (.assignmentInfo state storm-id nil)]
(->> assignment
- :executor->node+port
- vals
+ .get_executor_node_port
+ .values
set
)))
@@ -114,12 +121,12 @@
; map-val is a temporary kluge for clojure.
(defn topology-node-distribution [state storm-name]
(let [storm-id (StormCommon/getStormId state storm-name)
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))]
+ assignment (.assignmentInfo state storm-id nil)]
(->> assignment
- :executor->node+port
- vals
+ .get_executor_node_port
+ .values
set
- (group-by first)
+ (group-by (fn [np] (.get_node np)))
(map-val count)
(map (fn [[_ amt]] {amt 1}))
(apply merge-with +)
@@ -129,20 +136,22 @@
(count (topology-nodes state storm-name)))
(defn executor-assignment [cluster storm-id executor-id]
- (let [state (:storm-cluster-state cluster)
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))]
- ((:executor->node+port assignment) executor-id)
+ (let [state (.getClusterState cluster)
+ assignment (.assignmentInfo state storm-id nil)]
+ (.get (.get_executor_node_port assignment) executor-id)
))
(defn executor-start-times [cluster storm-id]
- (let [state (:storm-cluster-state cluster)
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))]
- (:executor->start-time-secs assignment)))
+ (let [state (.getClusterState cluster)
+ assignment (.assignmentInfo state storm-id nil)]
+ (clojurify-structure (.get_executor_start_time_secs assignment))))
(defn do-executor-heartbeat [cluster storm-id executor]
- (let [state (:storm-cluster-state cluster)
- executor->node+port (:executor->node+port (clojurify-assignment (.assignmentInfo state storm-id nil)))
- [node port] (get executor->node+port executor)
+ (let [state (.getClusterState cluster)
+ executor->node+port (.get_executor_node_port (.assignmentInfo state storm-id nil))
+ np (.get executor->node+port executor)
+ node (.get_node np)
+ port (first (.get_port np))
curr-beat (StatsUtil/convertZkWorkerHb (.getWorkerHeartbeat state storm-id node port))
stats (if (get curr-beat "executor-stats")
(get curr-beat "executor-stats")
@@ -156,28 +165,28 @@
(StatsUtil/thriftifyZkWorkerHb (StatsUtil/mkZkWorkerHb storm-id stats (int 10))))))
(defn slot-assignments [cluster storm-id]
- (let [state (:storm-cluster-state cluster)
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))]
- (clojurify-structure (Utils/reverseMap (:executor->node+port assignment)))))
+ (let [state (.getClusterState cluster)
+ assignment (.assignmentInfo state storm-id nil)]
+ (clojurify-structure (Utils/reverseMap (.get_executor_node_port assignment)))))
(defn task-ids [cluster storm-id]
- (let [nimbus (:nimbus cluster)]
+ (let [nimbus (.getNimbus cluster)]
(-> (.getUserTopology nimbus storm-id)
(#(StormCommon/stormTaskInfo % (from-json (.getTopologyConf nimbus storm-id))))
clojurify-structure
keys)))
(defn topology-executors [cluster storm-id]
- (let [state (:storm-cluster-state cluster)
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))
- ret-keys (keys (:executor->node+port assignment))
+ (let [state (.getClusterState cluster)
+ assignment (.assignmentInfo state storm-id nil)
+ ret-keys (keys (.get_executor_node_port assignment))
_ (log-message "ret-keys: " (pr-str ret-keys)) ]
ret-keys
))
(defn check-distribution [items distribution]
- (let [counts (map count items)]
- (is (ms= counts distribution))))
+ (let [counts (map long (map count items))]
+ (is (Testing/multiseteq counts (map long distribution)))))
(defn disjoint? [& sets]
(let [combined (apply concat sets)]
@@ -188,14 +197,14 @@
clojurify-structure (StormCommon/executorIdToTasks executor-id))
(defnk check-consistency [cluster storm-name :assigned? true]
- (let [state (:storm-cluster-state cluster)
+ (let [state (.getClusterState cluster)
storm-id (StormCommon/getStormId state storm-name)
task-ids (task-ids cluster storm-id)
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))
- executor->node+port (:executor->node+port assignment)
+ assignment (.assignmentInfo state storm-id nil)
+ executor->node+port (.get_executor_node_port assignment)
task->node+port (StormCommon/taskToNodeport executor->node+port)
assigned-task-ids (mapcat executor->tasks (keys executor->node+port))
- all-nodes (set (map first (vals executor->node+port)))]
+ all-nodes (set (map (fn [np] (.get_node np)) (.values executor->node+port)))]
(when assigned?
(is (= (sort task-ids) (sort assigned-task-ids)))
(doseq [t task-ids]
@@ -203,17 +212,17 @@
(doseq [[e s] executor->node+port]
(is (not-nil? s)))
- ;;(map str (-> (Thread/currentThread) .getStackTrace))
- (is (= all-nodes (set (keys (:node->host assignment)))))
+ (is (= all-nodes (set (keys (.get_node_host assignment)))))
(doseq [[e s] executor->node+port]
- (is (not-nil? ((:executor->start-time-secs assignment) e))))
+ (is (not-nil? (.get (.get_executor_start_time_secs assignment) e))))
))
(deftest test-bogusId
- (with-local-cluster [cluster :supervisors 4 :ports-per-supervisor 3
- :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
- (let [state (:storm-cluster-state cluster)
- nimbus (:nimbus cluster)]
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSupervisors 4)
+ (.withDaemonConf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
+ (let [state (.getClusterState cluster)
+ nimbus (.getNimbus cluster)]
(is (thrown? NotAliveException (.getTopologyConf nimbus "bogus-id")))
(is (thrown? NotAliveException (.getTopology nimbus "bogus-id")))
(is (thrown? NotAliveException (.getUserTopology nimbus "bogus-id")))
@@ -222,10 +231,11 @@
)))
(deftest test-assignment
- (with-simulated-time-local-cluster [cluster :supervisors 4 :ports-per-supervisor 3
- :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
- (let [state (:storm-cluster-state cluster)
- nimbus (:nimbus cluster)
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 4)
+ (.withDaemonConf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
+ (let [state (.getClusterState cluster)
topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. false) (Integer. 3))}
@@ -254,8 +264,8 @@
(Utils/getGlobalStreamId "2" nil)
(Thrift/prepareNoneGrouping)}
(TestPlannerBolt.) (Integer. 4))})
- _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-WORKERS 4} topology)
- _ (advance-cluster-time cluster 11)
+ _ (.submitTopology cluster "mystorm" {TOPOLOGY-WORKERS 4} topology)
+ _ (.advanceClusterTime cluster 11)
task-info (storm-component->task-info cluster "mystorm")]
(check-consistency cluster "mystorm")
;; 3 should be assigned once (if it were optimized, we'd have
@@ -265,8 +275,8 @@
(is (= 4 (count (task-info "2"))))
(is (= 1 (count (task-info "3"))))
(is (= 4 (storm-num-workers state "mystorm")))
- (submit-local-topology nimbus "storm2" {TOPOLOGY-WORKERS 20} topology2)
- (advance-cluster-time cluster 11)
+ (.submitTopology cluster "storm2" {TOPOLOGY-WORKERS 20} topology2)
+ (.advanceClusterTime cluster 11)
(check-consistency cluster "storm2")
(is (= 2 (count (.assignments state nil))))
(let [task-info (storm-component->task-info cluster "storm2")]
@@ -297,17 +307,17 @@
(deftest test-auto-credentials
- (with-simulated-time-local-cluster [cluster :supervisors 6
- :ports-per-supervisor 3
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 6)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
TOPOLOGY-ACKER-EXECUTORS 0
TOPOLOGY-EVENTLOGGER-EXECUTORS 0
NIMBUS-CREDENTIAL-RENEW-FREQ-SECS 10
NIMBUS-CREDENTIAL-RENEWERS (list "org.apache.storm.MockAutoCred")
NIMBUS-AUTO-CRED-PLUGINS (list "org.apache.storm.MockAutoCred")
- }]
- (let [state (:storm-cluster-state cluster)
- nimbus (:nimbus cluster)
+ })))]
+ (let [state (.getClusterState cluster)
topology-name "test-auto-cred-storm"
submitOptions (SubmitOptions. TopologyInitialStatus/INACTIVE)
- (.set_creds submitOptions (Credentials. (HashMap.)))
@@ -322,14 +332,14 @@
{(Utils/getGlobalStreamId "2" nil)
(Thrift/prepareNoneGrouping)}
(TestPlannerBolt.))})
- _ (submit-local-topology-with-opts nimbus topology-name {TOPOLOGY-WORKERS 4
+ _ (.submitTopologyWithOpts cluster topology-name {TOPOLOGY-WORKERS 4
TOPOLOGY-AUTO-CREDENTIALS (list "org.apache.storm.MockAutoCred")
} topology submitOptions)
credentials (getCredentials cluster topology-name)]
; check that the credentials have nimbus auto generated cred
(is (= (.get credentials MockAutoCred/NIMBUS_CRED_KEY) MockAutoCred/NIMBUS_CRED_VAL))
;advance cluster time so the renewers can execute
- (advance-cluster-time cluster 20)
+ (.advanceClusterTime cluster 20)
;check that renewed credentials replace the original credential.
(is (= (.get (getCredentials cluster topology-name) MockAutoCred/NIMBUS_CRED_KEY) MockAutoCred/NIMBUS_CRED_RENEW_VAL))
(is (= (.get (getCredentials cluster topology-name) MockAutoCred/GATEWAY_CRED_KEY) MockAutoCred/GATEWAY_CRED_RENEW_VAL)))))
@@ -347,19 +357,19 @@
~(first lexpr))))
(deftest test-isolated-assignment
- (with-simulated-time-local-cluster [cluster :supervisors 6
- :ports-per-supervisor 3
- :inimbus (isolation-nimbus)
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 6)
+ (.withINimbus (isolation-nimbus))
+ (.withDaemonConf {SUPERVISOR-ENABLE false
TOPOLOGY-ACKER-EXECUTORS 0
TOPOLOGY-EVENTLOGGER-EXECUTORS 0
STORM-SCHEDULER "org.apache.storm.scheduler.IsolationScheduler"
ISOLATION-SCHEDULER-MACHINES {"tester1" 3 "tester2" 2}
NIMBUS-MONITOR-FREQ-SECS 10
- }]
+ })))]
(letlocals
- (bind state (:storm-cluster-state cluster))
- (bind nimbus (:nimbus cluster))
+ (bind state (.getClusterState cluster))
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. false) (Integer. 3))}
@@ -372,14 +382,14 @@
(Thrift/prepareNoneGrouping)}
(TestPlannerBolt.))}))
- (submit-local-topology nimbus "noniso" {TOPOLOGY-WORKERS 4} topology)
- (advance-cluster-time cluster 11)
+ (.submitTopology cluster "noniso" {TOPOLOGY-WORKERS 4} topology)
+ (.advanceClusterTime cluster 11)
(is (= 4 (topology-num-nodes state "noniso")))
(is (= 4 (storm-num-workers state "noniso")))
- (submit-local-topology nimbus "tester1" {TOPOLOGY-WORKERS 6} topology)
- (submit-local-topology nimbus "tester2" {TOPOLOGY-WORKERS 6} topology)
- (advance-cluster-time cluster 11)
+ (.submitTopology cluster "tester1" {TOPOLOGY-WORKERS 6} topology)
+ (.submitTopology cluster "tester2" {TOPOLOGY-WORKERS 6} topology)
+ (.advanceClusterTime cluster 11)
(bind task-info-tester1 (storm-component->task-info cluster "tester1"))
(bind task-info-tester2 (storm-component->task-info cluster "tester2"))
@@ -401,7 +411,7 @@
(bind tester1-slots (topology-slots state "tester1"))
(bind tester2-slots (topology-slots state "tester2"))
(bind noniso-slots (topology-slots state "noniso"))
- (advance-cluster-time cluster 20)
+ (.advanceClusterTime cluster 20)
(is (= tester1-slots (topology-slots state "tester1")))
(is (= tester2-slots (topology-slots state "tester2")))
(is (= noniso-slots (topology-slots state "noniso")))
@@ -409,9 +419,11 @@
)))
(deftest test-zero-executor-or-tasks
- (with-simulated-time-local-cluster [cluster :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
- (let [state (:storm-cluster-state cluster)
- nimbus (:nimbus cluster)
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 6)
+ (.withDaemonConf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
+ (let [state (.getClusterState cluster)
topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. false) (Integer. 3)
@@ -426,8 +438,8 @@
(Thrift/prepareNoneGrouping)}
(TestPlannerBolt.) nil
{TOPOLOGY-TASKS 5})})
- _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-WORKERS 4} topology)
- _ (advance-cluster-time cluster 11)
+ _ (.submitTopology cluster "mystorm" {TOPOLOGY-WORKERS 4} topology)
+ _ (.advanceClusterTime cluster 11)
task-info (storm-component->task-info cluster "mystorm")]
(check-consistency cluster "mystorm")
(is (= 0 (count (task-info "1"))))
@@ -438,9 +450,10 @@
;TODO: when translating this function, you should replace the map-val with a proper for loop HERE
(deftest test-executor-assignments
- (with-simulated-time-local-cluster[cluster :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
- (let [nimbus (:nimbus cluster)
- topology (Thrift/buildTopology
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withDaemonConf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
+ (let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 3)
{TOPOLOGY-TASKS 5})}
@@ -453,8 +466,8 @@
{(Utils/getGlobalStreamId "2" nil)
(Thrift/prepareNoneGrouping)}
(TestPlannerBolt.) (Integer. 3))})
- _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-WORKERS 4} topology)
- _ (advance-cluster-time cluster 11)
+ _ (.submitTopology cluster "mystorm" {TOPOLOGY-WORKERS 4} topology)
+ _ (.advanceClusterTime cluster 11)
task-info (storm-component->task-info cluster "mystorm")
executor-info (->> (storm-component->executor-info cluster "mystorm")
(map-val #(map executor->tasks %)))]
@@ -470,10 +483,12 @@
)))
(deftest test-over-parallelism-assignment
- (with-simulated-time-local-cluster [cluster :supervisors 2 :ports-per-supervisor 5
- :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
- (let [state (:storm-cluster-state cluster)
- nimbus (:nimbus cluster)
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 2)
+ (.withPortsPerSupervisor 5)
+ (.withDaemonConf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
+ (let [state (.getClusterState cluster)
topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 21))}
@@ -489,8 +504,8 @@
{(Utils/getGlobalStreamId "1" nil)
(Thrift/prepareNoneGrouping)}
(TestPlannerBolt.) (Integer. 10))})
- _ (submit-local-topology nimbus "test" {TOPOLOGY-WORKERS 7} topology)
- _ (advance-cluster-time cluster 11)
+ _ (.submitTopology cluster "test" {TOPOLOGY-WORKERS 7} topology)
+ _ (.advanceClusterTime cluster 11)
task-info (storm-component->task-info cluster "test")]
(check-consistency cluster "test")
(is (= 21 (count (task-info "1"))))
@@ -502,81 +517,84 @@
(deftest test-topo-history
(let [group-mapper (Mockito/mock IGroupMappingServiceProvider)]
- (with-simulated-time-local-cluster [cluster :supervisors 2 :ports-per-supervisor 5
- :group-mapper group-mapper
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 2)
+ (.withPortsPerSupervisor 5)
+ (.withGroupMapper group-mapper)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
NIMBUS-ADMINS ["admin-user"]
NIMBUS-TASK-TIMEOUT-SECS 30
NIMBUS-MONITOR-FREQ-SECS 10
- TOPOLOGY-ACKER-EXECUTORS 0}]
+ TOPOLOGY-ACKER-EXECUTORS 0})))]
(.thenReturn (Mockito/when (.getGroups group-mapper (Mockito/anyObject))) #{"alice-group"})
(letlocals
- (bind conf (:daemon-conf cluster))
+ (bind conf (.getDaemonConf cluster))
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 4))}
{}))
- (bind state (:storm-cluster-state cluster))
- (submit-local-topology (:nimbus cluster) "test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20, LOGS-USERS ["alice", (System/getProperty "user.name")]} topology)
+ (bind state (.getClusterState cluster))
+ (.submitTopology cluster "test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20, LOGS-USERS ["alice", (System/getProperty "user.name")]} topology)
(bind storm-id (StormCommon/getStormId state "test"))
- (advance-cluster-time cluster 5)
- (is (not-nil? (clojurify-storm-base (.stormBase state storm-id nil))))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id nil))))
- (.killTopology (:nimbus cluster) "test")
+ (.advanceClusterTime cluster 5)
+ (is (not-nil? (.stormBase state storm-id nil)))
+ (is (not-nil? (.assignmentInfo state storm-id nil)))
+ (.killTopology (.getNimbus cluster) "test")
;; check that storm is deactivated but alive
- (is (= TopologyStatus/KILLED (-> (clojurify-storm-base (.stormBase state storm-id nil)) :status :type)))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id nil))))
- (advance-cluster-time cluster 35)
+ (is (= TopologyStatus/KILLED (.get_status (.stormBase state storm-id nil))))
+ (is (not-nil? (.assignmentInfo state storm-id nil)))
+ (.advanceClusterTime cluster 35)
;; kill topology read on group
- (submit-local-topology (:nimbus cluster) "killgrouptest" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20, LOGS-GROUPS ["alice-group"]} topology)
+ (.submitTopology cluster "killgrouptest" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20, LOGS-GROUPS ["alice-group"]} topology)
(bind storm-id-killgroup (StormCommon/getStormId state "killgrouptest"))
- (advance-cluster-time cluster 5)
- (is (not-nil? (clojurify-storm-base (.stormBase state storm-id-killgroup nil))))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id-killgroup nil))))
- (.killTopology (:nimbus cluster) "killgrouptest")
+ (.advanceClusterTime cluster 5)
+ (is (not-nil? (.stormBase state storm-id-killgroup nil)))
+ (is (not-nil? (.assignmentInfo state storm-id-killgroup nil)))
+ (.killTopology (.getNimbus cluster) "killgrouptest")
;; check that storm is deactivated but alive
- (is (= TopologyStatus/KILLED (-> (clojurify-storm-base (.stormBase state storm-id-killgroup nil)) :status :type)))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id-killgroup nil))))
- (advance-cluster-time cluster 35)
+ (is (= TopologyStatus/KILLED (.get_status (.stormBase state storm-id-killgroup nil))))
+ (is (not-nil? (.assignmentInfo state storm-id-killgroup nil)))
+ (.advanceClusterTime cluster 35)
;; kill topology can't read
- (submit-local-topology (:nimbus cluster) "killnoreadtest" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20} topology)
+ (.submitTopology cluster "killnoreadtest" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20} topology)
(bind storm-id-killnoread (StormCommon/getStormId state "killnoreadtest"))
- (advance-cluster-time cluster 5)
- (is (not-nil? (clojurify-storm-base (.stormBase state storm-id-killnoread nil))))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id-killnoread nil))))
- (.killTopology (:nimbus cluster) "killnoreadtest")
+ (.advanceClusterTime cluster 5)
+ (is (not-nil? (.stormBase state storm-id-killnoread nil)))
+ (is (not-nil? (.assignmentInfo state storm-id-killnoread nil)))
+ (.killTopology (.getNimbus cluster) "killnoreadtest")
;; check that storm is deactivated but alive
- (is (= TopologyStatus/KILLED (-> (clojurify-storm-base (.stormBase state storm-id-killnoread nil)) :status :type)))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id-killnoread nil))))
- (advance-cluster-time cluster 35)
+ (is (= TopologyStatus/KILLED (.get_status (.stormBase state storm-id-killnoread nil))))
+ (is (not-nil? (.assignmentInfo state storm-id-killnoread nil)))
+ (.advanceClusterTime cluster 35)
;; active topology can read
- (submit-local-topology (:nimbus cluster) "2test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10, LOGS-USERS ["alice", (System/getProperty "user.name")]} topology)
- (advance-cluster-time cluster 11)
+ (.submitTopology cluster "2test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10, LOGS-USERS ["alice", (System/getProperty "user.name")]} topology)
+ (.advanceClusterTime cluster 11)
(bind storm-id2 (StormCommon/getStormId state "2test"))
- (is (not-nil? (clojurify-storm-base (.stormBase state storm-id2 nil))))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id2 nil))))
+ (is (not-nil? (.stormBase state storm-id2 nil)))
+ (is (not-nil? (.assignmentInfo state storm-id2 nil)))
;; active topology can not read
- (submit-local-topology (:nimbus cluster) "testnoread" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10, LOGS-USERS ["alice"]} topology)
- (advance-cluster-time cluster 11)
+ (.submitTopology cluster "testnoread" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10, LOGS-USERS ["alice"]} topology)
+ (.advanceClusterTime cluster 11)
(bind storm-id3 (StormCommon/getStormId state "testnoread"))
- (is (not-nil? (clojurify-storm-base (.stormBase state storm-id3 nil))))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id3 nil))))
+ (is (not-nil? (.stormBase state storm-id3 nil)))
+ (is (not-nil? (.assignmentInfo state storm-id3 nil)))
;; active topology can read based on group
- (submit-local-topology (:nimbus cluster) "testreadgroup" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10, LOGS-GROUPS ["alice-group"]} topology)
- (advance-cluster-time cluster 11)
+ (.submitTopology cluster "testreadgroup" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10, LOGS-GROUPS ["alice-group"]} topology)
+ (.advanceClusterTime cluster 11)
(bind storm-id4 (StormCommon/getStormId state "testreadgroup"))
- (is (not-nil? (clojurify-storm-base (.stormBase state storm-id4 nil))))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id4 nil))))
+ (is (not-nil? (.stormBase state storm-id4 nil)))
+ (is (not-nil? (.assignmentInfo state storm-id4 nil)))
;; at this point have 1 running, 1 killed topo
- (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (:nimbus cluster) (System/getProperty "user.name")))))]
+ (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (.getNimbus cluster) (System/getProperty "user.name")))))]
(log-message "Checking user " (System/getProperty "user.name") " " hist-topo-ids)
(is (= 4 (count hist-topo-ids)))
(is (= storm-id2 (get hist-topo-ids 0)))
(is (= storm-id-killgroup (get hist-topo-ids 1)))
(is (= storm-id (get hist-topo-ids 2)))
(is (= storm-id4 (get hist-topo-ids 3))))
- (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (:nimbus cluster) "alice"))))]
+ (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (.getNimbus cluster) "alice"))))]
(log-message "Checking user alice " hist-topo-ids)
(is (= 5 (count hist-topo-ids)))
(is (= storm-id2 (get hist-topo-ids 0)))
@@ -584,7 +602,7 @@
(is (= storm-id (get hist-topo-ids 2)))
(is (= storm-id3 (get hist-topo-ids 3)))
(is (= storm-id4 (get hist-topo-ids 4))))
- (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (:nimbus cluster) "admin-user"))))]
+ (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (.getNimbus cluster) "admin-user"))))]
(log-message "Checking user admin-user " hist-topo-ids)
(is (= 6 (count hist-topo-ids)))
(is (= storm-id2 (get hist-topo-ids 0)))
@@ -593,122 +611,128 @@
(is (= storm-id (get hist-topo-ids 3)))
(is (= storm-id3 (get hist-topo-ids 4)))
(is (= storm-id4 (get hist-topo-ids 5))))
- (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (:nimbus cluster) "group-only-user"))))]
+ (let [hist-topo-ids (vec (sort (.get_topo_ids (.getTopologyHistory (.getNimbus cluster) "group-only-user"))))]
(log-message "Checking user group-only-user " hist-topo-ids)
(is (= 2 (count hist-topo-ids)))
(is (= storm-id-killgroup (get hist-topo-ids 0)))
(is (= storm-id4 (get hist-topo-ids 1))))))))
(deftest test-kill-storm
- (with-simulated-time-local-cluster [cluster :supervisors 2 :ports-per-supervisor 5
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 2)
+ (.withPortsPerSupervisor 5)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
NIMBUS-TASK-TIMEOUT-SECS 30
NIMBUS-MONITOR-FREQ-SECS 10
TOPOLOGY-ACKER-EXECUTORS 0
- TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
+ TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
(letlocals
- (bind conf (:daemon-conf cluster))
+ (bind conf (.getDaemonConf cluster))
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 14))}
{}))
- (bind state (:storm-cluster-state cluster))
- (submit-local-topology (:nimbus cluster) "test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20} topology)
+ (bind state (.getClusterState cluster))
+ (.submitTopology cluster "test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 20} topology)
(bind storm-id (StormCommon/getStormId state "test"))
- (advance-cluster-time cluster 15)
- (is (not-nil? (clojurify-storm-base (.stormBase state storm-id nil))))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id nil))))
- (.killTopology (:nimbus cluster) "test")
+ (.advanceClusterTime cluster 15)
+ (is (not-nil? (.stormBase state storm-id nil)))
+ (is (not-nil? (.assignmentInfo state storm-id nil)))
+ (.killTopology (.getNimbus cluster) "test")
;; check that storm is deactivated but alive
- (is (= TopologyStatus/KILLED (-> (clojurify-storm-base (.stormBase state storm-id nil)) :status :type)))
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id nil))))
- (advance-cluster-time cluster 18)
+ (is (= TopologyStatus/KILLED (.get_status (.stormBase state storm-id nil))))
+ (is (not-nil? (.assignmentInfo state storm-id nil)))
+ (.advanceClusterTime cluster 18)
;; check that storm is deactivated but alive
(is (= 1 (count (.heartbeatStorms state))))
- (advance-cluster-time cluster 3)
- (is (nil? (clojurify-storm-base (.stormBase state storm-id nil))))
- (is (nil? (clojurify-assignment (.assignmentInfo state storm-id nil))))
+ (.advanceClusterTime cluster 3)
+ (is (nil? (.stormBase state storm-id nil)))
+ (is (nil? (.assignmentInfo state storm-id nil)))
;; cleanup happens on monitoring thread
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(is (empty? (.heartbeatStorms state)))
;; TODO: check that code on nimbus was cleaned up locally...
- (is (thrown? NotAliveException (.killTopology (:nimbus cluster) "lalala")))
- (submit-local-topology (:nimbus cluster) "2test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10} topology)
- (advance-cluster-time cluster 11)
- (is (thrown? AlreadyAliveException (submit-local-topology (:nimbus cluster) "2test" {} topology)))
- (advance-cluster-time cluster 11)
+ (is (thrown? NotAliveException (.killTopology (.getNimbus cluster) "lalala")))
+ (.submitTopology cluster "2test" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10} topology)
+ (.advanceClusterTime cluster 11)
+ (is (thrown? AlreadyAliveException (.submitTopology cluster "2test" {} topology)))
+ (.advanceClusterTime cluster 11)
(bind storm-id (StormCommon/getStormId state "2test"))
- (is (not-nil? (clojurify-storm-base (.stormBase state storm-id nil))))
- (.killTopology (:nimbus cluster) "2test")
- (is (thrown? AlreadyAliveException (submit-local-topology (:nimbus cluster) "2test" {} topology)))
- (advance-cluster-time cluster 11)
+ (is (not-nil? (.stormBase state storm-id nil)))
+ (.killTopology (.getNimbus cluster) "2test")
+ (is (thrown? AlreadyAliveException (.submitTopology cluster "2test" {} topology)))
+ (.advanceClusterTime cluster 11)
(is (= 1 (count (.heartbeatStorms state))))
- (advance-cluster-time cluster 6)
- (is (nil? (clojurify-storm-base (.stormBase state storm-id nil))))
- (is (nil? (clojurify-assignment (.assignmentInfo state storm-id nil))))
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 6)
+ (is (nil? (.stormBase state storm-id nil)))
+ (is (nil? (.assignmentInfo state storm-id nil)))
+ (.advanceClusterTime cluster 11)
(is (= 0 (count (.heartbeatStorms state))))
- (submit-local-topology (:nimbus cluster) "test3" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 5} topology)
+ (.submitTopology cluster "test3" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 5} topology)
(bind storm-id3 (StormCommon/getStormId state "test3"))
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(.removeStorm state storm-id3)
- (is (nil? (clojurify-storm-base (.stormBase state storm-id3 nil))))
- (is (nil? (clojurify-assignment (.assignmentInfo state storm-id3 nil))))
+ (is (nil? (.stormBase state storm-id3 nil)))
+ (is (nil? (.assignmentInfo state storm-id3 nil)))
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(is (= 0 (count (.heartbeatStorms state))))
;; this guarantees that monitor thread won't trigger for 10 more seconds
- (advance-time-secs! 11)
- (wait-until-cluster-waiting cluster)
+ (Time/advanceTimeSecs 11)
+ (.waitForIdle cluster)
- (submit-local-topology (:nimbus cluster) "test3" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 5} topology)
+ (.submitTopology cluster "test3" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 5} topology)
(bind storm-id3 (StormCommon/getStormId state "test3"))
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(bind executor-id (first (topology-executors cluster storm-id3)))
(do-executor-heartbeat cluster storm-id3 executor-id)
- (.killTopology (:nimbus cluster) "test3")
- (advance-cluster-time cluster 6)
+ (.killTopology (.getNimbus cluster) "test3")
+ (.advanceClusterTime cluster 6)
(is (= 1 (count (.heartbeatStorms state))))
- (advance-cluster-time cluster 5)
+ (.advanceClusterTime cluster 5)
(is (= 0 (count (.heartbeatStorms state))))
;; test kill with opts
- (submit-local-topology (:nimbus cluster) "test4" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 100} topology)
- (advance-cluster-time cluster 11)
- (.killTopologyWithOpts (:nimbus cluster) "test4" (doto (KillOptions.) (.set_wait_secs 10)))
+ (.submitTopology cluster "test4" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 100} topology)
+ (.advanceClusterTime cluster 11)
+ (.killTopologyWithOpts (.getNimbus cluster) "test4" (doto (KillOptions.) (.set_wait_secs 10)))
(bind storm-id4 (StormCommon/getStormId state "test4"))
- (advance-cluster-time cluster 9)
- (is (not-nil? (clojurify-assignment (.assignmentInfo state storm-id4 nil))))
- (advance-cluster-time cluster 2)
- (is (nil? (clojurify-assignment (.assignmentInfo state storm-id4 nil))))
+ (.advanceClusterTime cluster 9)
+ (is (not-nil? (.assignmentInfo state storm-id4 nil)))
+ (.advanceClusterTime cluster 2)
+ (is (nil? (.assignmentInfo state storm-id4 nil)))
)))
(deftest test-reassignment
- (with-simulated-time-local-cluster [cluster :supervisors 2 :ports-per-supervisor 5
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 2)
+ (.withPortsPerSupervisor 5)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
NIMBUS-TASK-LAUNCH-SECS 60
NIMBUS-TASK-TIMEOUT-SECS 20
NIMBUS-MONITOR-FREQ-SECS 10
NIMBUS-SUPERVISOR-TIMEOUT-SECS 100
TOPOLOGY-ACKER-EXECUTORS 0
- TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
+ TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
(letlocals
- (bind conf (:daemon-conf cluster))
+ (bind conf (.getDaemonConf cluster))
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 2))}
{}))
- (bind state (:storm-cluster-state cluster))
- (submit-local-topology (:nimbus cluster) "test" {TOPOLOGY-WORKERS 2} topology)
- (advance-cluster-time cluster 11)
+ (bind state (.getClusterState cluster))
+ (.submitTopology cluster "test" {TOPOLOGY-WORKERS 2} topology)
+ (.advanceClusterTime cluster 11)
(check-consistency cluster "test")
(bind storm-id (StormCommon/getStormId state "test"))
(bind [executor-id1 executor-id2] (topology-executors cluster storm-id))
@@ -717,7 +741,7 @@
(bind _ (log-message "ass1, t0: " (pr-str ass1)))
(bind _ (log-message "ass2, t0: " (pr-str ass2)))
- (advance-cluster-time cluster 30)
+ (.advanceClusterTime cluster 30)
(bind _ (log-message "ass1, t30, pre beat: " (pr-str ass1)))
(bind _ (log-message "ass2, t30, pre beat: " (pr-str ass2)))
(do-executor-heartbeat cluster storm-id executor-id1)
@@ -725,7 +749,7 @@
(bind _ (log-message "ass1, t30, post beat: " (pr-str ass1)))
(bind _ (log-message "ass2, t30, post beat: " (pr-str ass2)))
- (advance-cluster-time cluster 13)
+ (.advanceClusterTime cluster 13)
(bind _ (log-message "ass1, t43, pre beat: " (pr-str ass1)))
(bind _ (log-message "ass2, t43, pre beat: " (pr-str ass2)))
(is (= ass1 (executor-assignment cluster storm-id executor-id1)))
@@ -734,7 +758,7 @@
(bind _ (log-message "ass1, t43, post beat: " (pr-str ass1)))
(bind _ (log-message "ass2, t43, post beat: " (pr-str ass2)))
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(bind _ (log-message "ass1, t54, pre beat: " (pr-str ass1)))
(bind _ (log-message "ass2, t54, pre beat: " (pr-str ass2)))
(do-executor-heartbeat cluster storm-id executor-id1)
@@ -745,7 +769,7 @@
; have to wait an extra 10 seconds because nimbus may not
; resynchronize its heartbeat time till monitor-time secs after
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(bind _ (log-message "ass1, t65, pre beat: " (pr-str ass1)))
(bind _ (log-message "ass2, t65, pre beat: " (pr-str ass2)))
(do-executor-heartbeat cluster storm-id executor-id1)
@@ -754,7 +778,7 @@
(is (= ass1 (executor-assignment cluster storm-id executor-id1)))
(check-consistency cluster "test")
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(bind _ (log-message "ass1, t76, pre beat: " (pr-str ass1)))
(bind _ (log-message "ass2, t76, pre beat: " (pr-str ass2)))
(is (= ass1 (executor-assignment cluster storm-id executor-id1)))
@@ -764,98 +788,100 @@
(bind _ (log-message "ass2, t76, post beat: " (pr-str ass2)))
(check-consistency cluster "test")
- (advance-cluster-time cluster 31)
+ (.advanceClusterTime cluster 31)
(is (not= ass1 (executor-assignment cluster storm-id executor-id1)))
(is (= ass2 (executor-assignment cluster storm-id executor-id2))) ; tests launch timeout
(check-consistency cluster "test")
(bind ass1 (executor-assignment cluster storm-id executor-id1))
- (bind active-supervisor (first ass2))
- (kill-supervisor cluster active-supervisor)
+ (bind active-supervisor (.get_node ass2))
+ (.killSupervisor cluster active-supervisor)
(doseq [i (range 12)]
(do-executor-heartbeat cluster storm-id executor-id1)
(do-executor-heartbeat cluster storm-id executor-id2)
- (advance-cluster-time cluster 10)
+ (.advanceClusterTime cluster 10)
)
;; tests that it doesn't reassign executors if they're heartbeating even if supervisor times out
(is (= ass1 (executor-assignment cluster storm-id executor-id1)))
(is (= ass2 (executor-assignment cluster storm-id executor-id2)))
(check-consistency cluster "test")
- (advance-cluster-time cluster 30)
+ (.advanceClusterTime cluster 30)
(bind ass1 (executor-assignment cluster storm-id executor-id1))
(bind ass2 (executor-assignment cluster storm-id executor-id2))
(is (not-nil? ass1))
(is (not-nil? ass2))
- (is (not= active-supervisor (first (executor-assignment cluster storm-id executor-id2))))
- (is (not= active-supervisor (first (executor-assignment cluster storm-id executor-id1))))
+ (is (not= active-supervisor (.get_node (executor-assignment cluster storm-id executor-id2))))
+ (is (not= active-supervisor (.get_node (executor-assignment cluster storm-id executor-id1))))
(check-consistency cluster "test")
(doseq [supervisor-id (.supervisors state nil)]
- (kill-supervisor cluster supervisor-id))
+ (.killSupervisor cluster supervisor-id))
- (advance-cluster-time cluster 90)
+ (.advanceClusterTime cluster 90)
(bind ass1 (executor-assignment cluster storm-id executor-id1))
(bind ass2 (executor-assignment cluster storm-id executor-id2))
(is (nil? ass1))
(is (nil? ass2))
(check-consistency cluster "test" :assigned? false)
- (add-supervisor cluster)
- (advance-cluster-time cluster 11)
+ (.addSupervisor cluster)
+ (.advanceClusterTime cluster 11)
(check-consistency cluster "test")
)))
(deftest test-reassignment-to-constrained-cluster
- (with-simulated-time-local-cluster [cluster :supervisors 0
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 0)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
NIMBUS-TASK-LAUNCH-SECS 60
NIMBUS-TASK-TIMEOUT-SECS 20
NIMBUS-MONITOR-FREQ-SECS 10
NIMBUS-SUPERVISOR-TIMEOUT-SECS 100
TOPOLOGY-ACKER-EXECUTORS 0
- TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
+ TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
(letlocals
- (add-supervisor cluster :ports 1 :id "a")
- (add-supervisor cluster :ports 1 :id "b")
- (bind conf (:daemon-conf cluster))
+ (.addSupervisor cluster 1 "a")
+ (.addSupervisor cluster 1 "b")
+ (bind conf (.getDaemonConf cluster))
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 2))}
{}))
- (bind state (:storm-cluster-state cluster))
- (submit-local-topology (:nimbus cluster) "test" {TOPOLOGY-WORKERS 2} topology)
- (advance-cluster-time cluster 11)
+ (bind state (.getClusterState cluster))
+ (.submitTopology cluster "test" {TOPOLOGY-WORKERS 2} topology)
+ (.advanceClusterTime cluster 11)
(check-consistency cluster "test")
(bind storm-id (StormCommon/getStormId state "test"))
(bind [executor-id1 executor-id2] (topology-executors cluster storm-id))
(bind ass1 (executor-assignment cluster storm-id executor-id1))
(bind ass2 (executor-assignment cluster storm-id executor-id2))
- (advance-cluster-time cluster 30)
+ (.advanceClusterTime cluster 30)
(do-executor-heartbeat cluster storm-id executor-id1)
(do-executor-heartbeat cluster storm-id executor-id2)
- (advance-cluster-time cluster 13)
+ (.advanceClusterTime cluster 13)
(is (= ass1 (executor-assignment cluster storm-id executor-id1)))
(is (= ass2 (executor-assignment cluster storm-id executor-id2)))
- (kill-supervisor cluster "b")
+ (.killSupervisor cluster "b")
(do-executor-heartbeat cluster storm-id executor-id1)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(do-executor-heartbeat cluster storm-id executor-id1)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(do-executor-heartbeat cluster storm-id executor-id1)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(do-executor-heartbeat cluster storm-id executor-id1)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(do-executor-heartbeat cluster storm-id executor-id1)
(check-consistency cluster "test")
@@ -866,42 +892,45 @@
(check-distribution (vals slot-executors) distribution))
(defn check-num-nodes [slot-executors num-nodes]
- (let [nodes (->> slot-executors keys (map first) set)]
+ (let [nodes (->> slot-executors keys (map (fn [np] (.get_node np))) set)]
(is (= num-nodes (count nodes)))
))
(deftest test-reassign-squeezed-topology
- (with-simulated-time-local-cluster [cluster :supervisors 1 :ports-per-supervisor 1
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 1)
+ (.withPortsPerSupervisor 1)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
NIMBUS-TASK-LAUNCH-SECS 60
NIMBUS-TASK-TIMEOUT-SECS 20
NIMBUS-MONITOR-FREQ-SECS 10
TOPOLOGY-ACKER-EXECUTORS 0
- TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
+ TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
(letlocals
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 9))}
{}))
- (bind state (:storm-cluster-state cluster))
- (submit-local-topology (:nimbus cluster) "test" {TOPOLOGY-WORKERS 4} topology) ; distribution should be 2, 2, 2, 3 ideally
- (advance-cluster-time cluster 11)
+ (bind state (.getClusterState cluster))
+ (.submitTopology cluster "test" {TOPOLOGY-WORKERS 4} topology) ; distribution should be 2, 2, 2, 3 ideally
+ (.advanceClusterTime cluster 11)
(bind storm-id (StormCommon/getStormId state "test"))
(bind slot-executors (slot-assignments cluster storm-id))
(check-executor-distribution slot-executors [9])
(check-consistency cluster "test")
- (add-supervisor cluster :ports 2)
- (advance-cluster-time cluster 11)
+ (.addSupervisor cluster 2)
+ (.advanceClusterTime cluster 11)
(bind slot-executors (slot-assignments cluster storm-id))
(bind executor->start (executor-start-times cluster storm-id))
(check-executor-distribution slot-executors [3 3 3])
(check-consistency cluster "test")
- (add-supervisor cluster :ports 8)
+ (.addSupervisor cluster 8)
;; this actually works for any time > 0, since zookeeper fires an event causing immediate reassignment
;; doesn't work for time = 0 because it's not waiting for cluster yet, so test might happen before reassignment finishes
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(bind slot-executors2 (slot-assignments cluster storm-id))
(bind executor->start2 (executor-start-times cluster storm-id))
(check-executor-distribution slot-executors2 [2 2 2 3])
@@ -922,47 +951,49 @@
)))
(deftest test-rebalance
- (with-simulated-time-local-cluster [cluster :supervisors 1 :ports-per-supervisor 3
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 1)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
NIMBUS-MONITOR-FREQ-SECS 10
TOPOLOGY-MESSAGE-TIMEOUT-SECS 30
TOPOLOGY-ACKER-EXECUTORS 0
- TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
+ TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
(letlocals
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 3))}
{}))
- (bind state (:storm-cluster-state cluster))
- (submit-local-topology (:nimbus cluster)
+ (bind state (.getClusterState cluster))
+ (.submitTopology cluster
"test"
{TOPOLOGY-WORKERS 3
TOPOLOGY-MESSAGE-TIMEOUT-SECS 60} topology)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(bind storm-id (StormCommon/getStormId state "test"))
- (add-supervisor cluster :ports 3)
- (add-supervisor cluster :ports 3)
+ (.addSupervisor cluster 3)
+ (.addSupervisor cluster 3)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(bind slot-executors (slot-assignments cluster storm-id))
;; check that all workers are on one machine
(check-executor-distribution slot-executors [1 1 1])
(check-num-nodes slot-executors 1)
- (.rebalance (:nimbus cluster) "test" (RebalanceOptions.))
+ (.rebalance (.getNimbus cluster) "test" (RebalanceOptions.))
- (advance-cluster-time cluster 30)
+ (.advanceClusterTime cluster 30)
(check-executor-distribution slot-executors [1 1 1])
(check-num-nodes slot-executors 1)
- (advance-cluster-time cluster 30)
+ (.advanceClusterTime cluster 30)
(bind slot-executors (slot-assignments cluster storm-id))
(check-executor-distribution slot-executors [1 1 1])
(check-num-nodes slot-executors 3)
(is (thrown? InvalidTopologyException
- (.rebalance (:nimbus cluster) "test"
+ (.rebalance (.getNimbus cluster) "test"
(doto (RebalanceOptions.)
(.set_num_executors {"1" (int 0)})
))))
@@ -970,23 +1001,25 @@
;TODO: when translating this function, you should replace the map-val with a proper for loop HERE
(deftest test-rebalance-change-parallelism
- (with-simulated-time-local-cluster [cluster :supervisors 4 :ports-per-supervisor 3
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 4)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
NIMBUS-MONITOR-FREQ-SECS 10
TOPOLOGY-ACKER-EXECUTORS 0
- TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
+ TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
(letlocals
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 6)
{TOPOLOGY-TASKS 12})}
{}))
- (bind state (:storm-cluster-state cluster))
- (submit-local-topology (:nimbus cluster)
+ (bind state (.getClusterState cluster))
+ (.submitTopology cluster
"test"
{TOPOLOGY-WORKERS 3
TOPOLOGY-MESSAGE-TIMEOUT-SECS 30} topology)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(bind storm-id (StormCommon/getStormId state "test"))
(bind checker (fn [distribution]
(check-executor-distribution
@@ -994,30 +1027,30 @@
distribution)))
(checker [2 2 2])
- (.rebalance (:nimbus cluster) "test"
+ (.rebalance (.getNimbus cluster) "test"
(doto (RebalanceOptions.)
(.set_num_workers (int 6))
))
- (advance-cluster-time cluster 29)
+ (.advanceClusterTime cluster 29)
(checker [2 2 2])
- (advance-cluster-time cluster 3)
+ (.advanceClusterTime cluster 3)
(checker [1 1 1 1 1 1])
- (.rebalance (:nimbus cluster) "test"
+ (.rebalance (.getNimbus cluster) "test"
(doto (RebalanceOptions.)
(.set_num_executors {"1" (int 1)})
))
- (advance-cluster-time cluster 29)
+ (.advanceClusterTime cluster 29)
(checker [1 1 1 1 1 1])
- (advance-cluster-time cluster 3)
+ (.advanceClusterTime cluster 3)
(checker [1])
- (.rebalance (:nimbus cluster) "test"
+ (.rebalance (.getNimbus cluster) "test"
(doto (RebalanceOptions.)
(.set_num_executors {"1" (int 8)})
(.set_num_workers 4)
))
- (advance-cluster-time cluster 32)
+ (.advanceClusterTime cluster 32)
(checker [2 2 2 2])
(check-consistency cluster "test")
@@ -1033,9 +1066,9 @@
(let [assignments (.assignments state nil)]
(log-message "Assignemts: " assignments)
(let [id->node->ports (into {} (for [id assignments
- :let [executor->node+port (:executor->node+port (clojurify-assignment (.assignmentInfo state id nil)))
+ :let [executor->node+port (.get_executor_node_port (.assignmentInfo state id nil))
node+ports (set (.values executor->node+port))
- node->ports (apply merge-with (fn [a b] (distinct (concat a b))) (for [[node port] node+ports] {node [port]}))]]
+ node->ports (apply merge-with (fn [a b] (distinct (concat a b))) (for [np node+ports] {(.get_node np) [(first (.get_port np))]}))]]
{id node->ports}))
_ (log-message "id->node->ports: " id->node->ports)
all-nodes (apply merge-with (fn [a b]
@@ -1047,12 +1080,15 @@
)))
(deftest test-rebalance-constrained-cluster
- (with-simulated-time-local-cluster [cluster :supervisors 1 :ports-per-supervisor 4
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withSupervisors 1)
+ (.withPortsPerSupervisor 4)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
NIMBUS-MONITOR-FREQ-SECS 10
TOPOLOGY-MESSAGE-TIMEOUT-SECS 30
TOPOLOGY-ACKER-EXECUTORS 0
- TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
+ TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
(letlocals
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
@@ -1066,43 +1102,44 @@
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 3))}
{}))
- (bind state (:storm-cluster-state cluster))
- (submit-local-topology (:nimbus cluster)
+ (bind state (.getClusterState cluster))
+ (.submitTopology cluster
"test"
{TOPOLOGY-WORKERS 3
TOPOLOGY-MESSAGE-TIMEOUT-SECS 90} topology)
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"test2"
{TOPOLOGY-WORKERS 3
TOPOLOGY-MESSAGE-TIMEOUT-SECS 90} topology2)
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"test3"
{TOPOLOGY-WORKERS 3
TOPOLOGY-MESSAGE-TIMEOUT-SECS 90} topology3)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(check-for-collisions state)
- (.rebalance (:nimbus cluster) "test" (doto (RebalanceOptions.)
+ (.rebalance (.getNimbus cluster) "test" (doto (RebalanceOptions.)
(.set_num_workers 4)
(.set_wait_secs 0)
))
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(check-for-collisions state)
- (advance-cluster-time cluster 30)
+ (.advanceClusterTime cluster 30)
(check-for-collisions state)
)))
(deftest test-submit-invalid
- (with-simulated-time-local-cluster [cluster
- :daemon-conf {SUPERVISOR-ENABLE false
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withSimulatedTime)
+ (.withDaemonConf {SUPERVISOR-ENABLE false
TOPOLOGY-ACKER-EXECUTORS 0
TOPOLOGY-EVENTLOGGER-EXECUTORS 0
NIMBUS-EXECUTORS-PER-TOPOLOGY 8
- NIMBUS-SLOTS-PER-TOPOLOGY 8}]
+ NIMBUS-SLOTS-PER-TOPOLOGY 8})))]
(letlocals
(bind topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
@@ -1110,7 +1147,7 @@
{TOPOLOGY-TASKS 1})}
{}))
(is (thrown? InvalidTopologyException
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"test/aaa"
{}
topology)))
@@ -1119,9 +1156,9 @@
(TestPlannerSpout. true) (Integer. 16)
{TOPOLOGY-TASKS 16})}
{}))
- (bind state (:storm-cluster-state cluster))
+ (bind state (.getClusterState cluster))
(is (thrown? InvalidTopologyException
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"test"
{TOPOLOGY-WORKERS 3}
topology)))
@@ -1131,67 +1168,67 @@
{TOPOLOGY-TASKS 5})}
{}))
(is (thrown? InvalidTopologyException
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"test"
{TOPOLOGY-WORKERS 16}
- topology)))
- (is (nil? (submit-local-topology (:nimbus cluster)
- "test"
- {TOPOLOGY-WORKERS 8}
- topology))))))
+ topology))))))
(deftest test-clean-inbox
"Tests that the inbox correctly cleans jar files."
- (with-simulated-time
- (with-local-tmp [dir-location]
- (let [dir (File. dir-location)
- mk-file (fn [name seconds-ago]
- (let [f (File. (str dir-location "/" name))
- t (- (Time/currentTimeMillis) (* seconds-ago 1000))]
- (FileUtils/touch f)
- (.setLastModified f t)))
- assert-files-in-dir (fn [compare-file-names]
- (let [file-names (map #(.getName %) (file-seq dir))]
- (is (= (sort compare-file-names)
- (sort (filter #(.endsWith % ".jar") file-names))
- ))))]
- ;; Make three files a.jar, b.jar, c.jar.
- ;; a and b are older than c and should be deleted first.
- (advance-time-secs! 100)
- (doseq [fs [["a.jar" 20] ["b.jar" 20] ["c.jar" 0]]]
- (apply mk-file fs))
- (assert-files-in-dir ["a.jar" "b.jar" "c.jar"])
- (Nimbus/cleanInbox dir-location 10)
- (assert-files-in-dir ["c.jar"])
- ;; Cleanit again, c.jar should stay
- (advance-time-secs! 5)
- (Nimbus/cleanInbox dir-location 10)
- (assert-files-in-dir ["c.jar"])
- ;; Advance time, clean again, c.jar should be deleted.
- (advance-time-secs! 5)
- (Nimbus/cleanInbox dir-location 10)
- (assert-files-in-dir [])
- ))))
+ (with-open [_ (Time$SimulatedTime.)
+ tmp-path (TmpPath. )]
+ (let [dir-location (.getPath tmp-path)
+ dir (File. dir-location)
+ mk-file (fn [name seconds-ago]
+ (let [f (File. (str dir-location "/" name))
+ t (- (Time/currentTimeMillis) (* seconds-ago 1000))]
+ (FileUtils/touch f)
+ (.setLastModified f t)))
+ assert-files-in-dir (fn [compare-file-names]
+ (let [file-names (map #(.getName %) (file-seq dir))]
+ (is (= (sort compare-file-names)
+ (sort (filter #(.endsWith % ".jar") file-names))
+ ))))]
+ ;; Make three files a.jar, b.jar, c.jar.
+ ;; a and b are older than c and should be deleted first.
+ (Time/advanceTimeSecs 100)
+ (doseq [fs [["a.jar" 20] ["b.jar" 20] ["c.jar" 0]]]
+ (apply mk-file fs))
+ (assert-files-in-dir ["a.jar" "b.jar" "c.jar"])
+ (Nimbus/cleanInbox dir-location 10)
+ (assert-files-in-dir ["c.jar"])
+ ;; Cleanit again, c.jar should stay
+ (Time/advanceTimeSecs 5)
+ (Nimbus/cleanInbox dir-location 10)
+ (assert-files-in-dir ["c.jar"])
+ ;; Advance time, clean again, c.jar should be deleted.
+ (Time/advanceTimeSecs 5)
+ (Nimbus/cleanInbox dir-location 10)
+ (assert-files-in-dir [])
+ )))
(defn wait-for-status [nimbus name status]
- (while-timeout 5000
- (let [topo-summary (first (filter (fn [topo] (= name (.get_name topo))) (.get_topologies (.getClusterInfo nimbus))))
- topo-status (if topo-summary (.get_status topo-summary) "NOT-RUNNING")]
- (log-message "WAITING FOR "name" TO BE " status " CURRENT " topo-status)
- (not= topo-status status))
- (Thread/sleep 100)))
+ (Testing/whileTimeout 5000
+ (reify Testing$Condition
+ (exec [this]
+ (let [topo-summary (first (filter (fn [topo] (= name (.get_name topo))) (.get_topologies (.getClusterInfo nimbus))))
+ topo-status (if topo-summary (.get_status topo-summary) "NOT-RUNNING")]
+ (log-message "WAITING FOR "name" TO BE " status " CURRENT " topo-status)
+ (not= topo-status status))))
+ (fn [] (Thread/sleep 100))))
(deftest test-leadership
"Tests that leader actions can only be performed by master and non leader fails to perform the same actions."
- (with-inprocess-zookeeper zk-port
- (with-local-tmp [nimbus-dir]
- (with-open [_ (MockedZookeeper. (proxy [Zookeeper] []
- (zkLeaderElectorImpl [conf blob-store] (mock-leader-elector))))]
+ (with-open [zk (InProcessZookeeper. )]
+ (with-open [tmp-nimbus-dir (TmpPath.)
+ _ (MockedZookeeper. (proxy [Zookeeper] []
+ (zkLeaderElectorImpl [conf blob-store] (MockLeaderElector. ))))]
+ (let [nimbus-dir (.getPath tmp-nimbus-dir)]
(letlocals
(bind conf (merge (clojurify-structure (ConfigUtils/readStormConfig))
{STORM-ZOOKEEPER-SERVERS ["localhost"]
STORM-CLUSTER-MODE "local"
- STORM-ZOOKEEPER-PORT zk-port
+ STORM-ZOOKEEPER-PORT (.getPort zk)
STORM-LOCAL-DIR nimbus-dir}))
(bind cluster-state (ClusterUtils/mkStormClusterState conf nil (ClusterStateContext.)))
(bind nimbus (mk-nimbus conf (Nimbus$StandaloneINimbus.) nil nil nil nil))
@@ -1202,7 +1239,7 @@
{}))
(with-open [_ (MockedZookeeper. (proxy [Zookeeper] []
- (zkLeaderElectorImpl [conf blob-store] (mock-leader-elector :is-leader false))))]
+ (zkLeaderElectorImpl [conf blob-store] (MockLeaderElector. false))))]
(letlocals
(bind non-leader-cluster-state (ClusterUtils/mkStormClusterState conf nil (ClusterStateContext.)))
@@ -1210,7 +1247,7 @@
(.launchServer non-leader-nimbus)
;first we verify that the master nimbus can perform all actions, even with another nimbus present.
- (submit-local-topology nimbus "t1" {} topology)
+ (.submitTopology nimbus "t1" nil "{}" topology)
;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
(.rebalance nimbus "t1" (doto (RebalanceOptions.) (.set_wait_secs 0)))
(wait-for-status nimbus "t1" "ACTIVE")
@@ -1221,9 +1258,10 @@
;now we verify that non master nimbus can not perform any of the actions.
(is (thrown? RuntimeException
- (submit-local-topology non-leader-nimbus
+ (.submitTopology non-leader-nimbus
"failing"
- {}
+ nil
+ "{}"
topology)))
(is (thrown? RuntimeException
@@ -1245,15 +1283,14 @@
(.disconnect cluster-state))))))
(deftest test-nimbus-iface-submitTopologyWithOpts-checks-authorization
- (with-local-cluster [cluster
- :daemon-conf {NIMBUS-AUTHORIZER
- "org.apache.storm.security.auth.authorizer.DenyAuthorizer"}]
+ (with-open [cluster (.build (doto (LocalCluster$Builder. )
+ (.withDaemonConf {NIMBUS-AUTHORIZER
+ "org.apache.storm.security.auth.authorizer.DenyAuthorizer"})))]
(let [
- nimbus (:nimbus cluster)
topology (Thrift/buildTopology {} {})
]
(is (thrown? AuthorizationException
- (submit-local-topology-with-opts nimbus "mystorm" {} topology
+ (.submitTopologyWithOpts cluster "mystorm" {} topology
(SubmitOptions. TopologyInitialStatus/INACTIVE))
))
)
@@ -1263,9 +1300,12 @@
(deftest test-nimbus-iface-methods-check-authorization
(let [cluster-state (Mockito/mock IStormClusterState)
blob-store (Mockito/mock BlobStore)]
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store
- :daemon-conf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.DenyAuthorizer"}]
- (let [nimbus (:nimbus cluster)
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder. )
+ (.withClusterState cluster-state)
+ (.withBlobStore blob-store)
+ (.withDaemonConf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.DenyAuthorizer"})))]
+ (let [nimbus (.getNimbus cluster)
topology-name "test"
topology-id "test-id"]
(.thenReturn (Mockito/when (.getTopoId cluster-state topology-name)) (Optional/of topology-id))
@@ -1278,13 +1318,14 @@
(deftest test-nimbus-check-authorization-params
(let [cluster-state (Mockito/mock IStormClusterState)
- blob-store (Mockito/mock BlobStore)
- mk-nimbus (fn
- [conf inimbus blob-store leader-elector group-mapper cluster-state]
- (Mockito/spy (mk-nimbus conf inimbus blob-store leader-elector group-mapper cluster-state)))]
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store :mk-nimbus mk-nimbus
- :daemon-conf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"}]
- (let [nimbus (:nimbus cluster)
+ blob-store (Mockito/mock BlobStore)]
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder. )
+ (.withClusterState cluster-state)
+ (.withBlobStore blob-store)
+ (.withNimbusWrapper (reify UnaryOperator (apply [this nimbus] (Mockito/spy nimbus))))
+ (.withDaemonConf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"})))]
+ (let [nimbus (.getNimbus cluster)
topology-name "test-nimbus-check-autho-params"
topology-id "fake-id"
topology (Thrift/buildTopology {} {})
@@ -1334,13 +1375,14 @@
(deftest test-check-authorization-getSupervisorPageInfo
(let [cluster-state (Mockito/mock IStormClusterState)
- blob-store (Mockito/mock BlobStore)
- mk-nimbus (fn
- [conf inimbus blob-store leader-elector group-mapper cluster-state]
- (Mockito/spy (mk-nimbus conf inimbus blob-store leader-elector group-mapper cluster-state)))]
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store :mk-nimbus mk-nimbus
- :daemon-conf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"}]
- (let [nimbus (:nimbus cluster)
+ blob-store (Mockito/mock BlobStore)]
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder. )
+ (.withClusterState cluster-state)
+ (.withBlobStore blob-store)
+ (.withNimbusWrapper (reify UnaryOperator (apply [this nimbus] (Mockito/spy nimbus))))
+ (.withDaemonConf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"})))]
+ (let [nimbus (.getNimbus cluster)
expected-name "test-nimbus-check-autho-params"
expected-conf {TOPOLOGY-NAME expected-name
TOPOLOGY-WORKERS 1
@@ -1354,7 +1396,6 @@
(.set_spouts {})
(.set_bolts {})
(.set_state_spouts {}))
- clojurified-assignment (clojurify-assignment assignment)
topo-assignment {expected-name assignment}
check-auth-state (atom [])
mock-check-authorization (fn [nimbus storm-name storm-conf operation]
@@ -1371,17 +1412,16 @@
(.thenReturn (Mockito/when (.readTopologyConf blob-store (Mockito/any String) (Mockito/any Subject))) expected-conf)
(.thenReturn (Mockito/when (.readTopology blob-store (Mockito/any String) (Mockito/any Subject))) topology)
(.thenReturn (Mockito/when (.topologyAssignments cluster-state)) topo-assignment)
- (stubbing [clojurify-assignment clojurified-assignment]
- (.getSupervisorPageInfo nimbus "super1" nil true)
+ (.getSupervisorPageInfo nimbus "super1" nil true)
- ;; afterwards, it should get called twice
- (.checkAuthorization (Mockito/verify (:nimbus cluster)) expected-name expected-conf "getSupervisorPageInfo")
- (.checkAuthorization (Mockito/verify (:nimbus cluster)) expected-name expected-conf "getTopology"))))))
+ ;; afterwards, it should get called twice
+ (.checkAuthorization (Mockito/verify nimbus) expected-name expected-conf "getSupervisorPageInfo")
+ (.checkAuthorization (Mockito/verify nimbus) expected-name expected-conf "getTopology")))))
(deftest test-nimbus-iface-getTopology-methods-throw-correctly
- (with-local-cluster [cluster]
+ (with-open [cluster (LocalCluster. )]
(let [
- nimbus (:nimbus cluster)
+ nimbus (.getNimbus cluster)
id "bogus ID"
]
(is (thrown? NotAliveException (.getTopology nimbus id)))
@@ -1416,22 +1456,27 @@
)
)
+(defn mkStormBase [launch-time-secs storm-name status]
+ (doto (StormBase.)
+ (.set_name storm-name)
+ (.set_launch_time_secs (int launch-time-secs))
+ (.set_status status)))
+
(deftest test-nimbus-iface-getClusterInfo-filters-topos-without-bases
(let [cluster-state (Mockito/mock IStormClusterState)
blob-store (Mockito/mock BlobStore)]
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store]
- (let [nimbus (:nimbus cluster)
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder. )
+ (.withClusterState cluster-state)
+ (.withBlobStore blob-store)))]
+ (let [nimbus (.getNimbus cluster)
bogus-secs 42
bogus-type TopologyStatus/ACTIVE
bogus-bases {
"1" nil
- "2" (thriftify-storm-base {:launch-time-secs bogus-secs
- :storm-name "id2-name"
- :status {:type bogus-type}})
+ "2" (mkStormBase bogus-secs "id2-name" bogus-type)
"3" nil
- "4" (thriftify-storm-base {:launch-time-secs bogus-secs
- :storm-name "id4-name"
- :status {:type bogus-type}})
+ "4" (mkStormBase bogus-secs "id4-name" bogus-type)
}
topo-name "test-topo"
topo-conf {TOPOLOGY-NAME topo-name
@@ -1499,8 +1544,10 @@
))))
(deftest test-file-bogus-download
- (with-local-cluster [cluster :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
- (let [nimbus (:nimbus cluster)]
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder. )
+ (.withDaemonConf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0})))]
+ (let [nimbus (.getNimbus cluster)]
(is (thrown-cause? AuthorizationException (.beginFileDownload nimbus nil)))
(is (thrown-cause? AuthorizationException (.beginFileDownload nimbus "")))
(is (thrown-cause? AuthorizationException (.beginFileDownload nimbus "/bogus-path/foo")))
@@ -1509,25 +1556,28 @@
(deftest test-validate-topo-config-on-submit
(let [cluster-state (Mockito/mock IStormClusterState)
blob-store (Mockito/mock BlobStore)]
- (.thenReturn (Mockito/when (.getTopoId cluster-state "test")) (Optional/empty))
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store
- :daemon-conf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"}]
- (let [nimbus (:nimbus cluster)
- topology (Thrift/buildTopology {} {})
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder. )
+ (.withClusterState cluster-state)
+ (.withBlobStore blob-store)
+ (.withDaemonConf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"})))]
+ (.thenReturn (Mockito/when (.getTopoId cluster-state "test")) (Optional/empty))
+ (let [topology (Thrift/buildTopology {} {})
bad-config {"topology.isolate.machines" "2"}]
(is (thrown-cause? InvalidTopologyException
- (submit-local-topology-with-opts nimbus "test" bad-config topology
+ (.submitTopologyWithOpts cluster "test" bad-config topology
(SubmitOptions.))))))))
(deftest test-stateless-with-scheduled-topology-to-be-killed
; tests regression of STORM-856
- (with-inprocess-zookeeper zk-port
- (with-local-tmp [nimbus-dir]
+ (with-open [zk (InProcessZookeeper. )]
+ (with-open [tmp-nimbus-dir (TmpPath. )]
+ (let [nimbus-dir (.getPath tmp-nimbus-dir)]
(letlocals
(bind conf (merge (clojurify-structure (ConfigUtils/readStormConfig))
{STORM-ZOOKEEPER-SERVERS ["localhost"]
STORM-CLUSTER-MODE "local"
- STORM-ZOOKEEPER-PORT zk-port
+ STORM-ZOOKEEPER-PORT (.getPort zk)
STORM-LOCAL-DIR nimbus-dir}))
(bind cluster-state (ClusterUtils/mkStormClusterState conf nil (ClusterStateContext.)))
(bind nimbus (mk-nimbus conf (Nimbus$StandaloneINimbus.) nil nil nil nil))
@@ -1537,7 +1587,7 @@
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 3))}
{}))
- (submit-local-topology nimbus "t1" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 30} topology)
+ (.submitTopology nimbus "t1" nil (str "{\"" TOPOLOGY-MESSAGE-TIMEOUT-SECS "\": 30}") topology)
; make transition for topology t1 to be killed -> nimbus applies this event to cluster state
(.killTopology nimbus "t1")
; shutdown nimbus immediately to achieve nimbus doesn't handle event right now
@@ -1550,18 +1600,19 @@
(.launchServer nimbus)
(.shutdown nimbus)
(.disconnect cluster-state)
- ))))
+ )))))
(deftest test-topology-action-notifier
- (with-inprocess-zookeeper zk-port
- (with-local-tmp [nimbus-dir]
- (with-open [_ (MockedZookeeper. (proxy [Zookeeper] []
- (zkLeaderElectorImpl [conf blob-store] (mock-leader-elector))))]
+ (with-open [zk (InProcessZookeeper. )]
+ (with-open [tmp-nimbus-dir (TmpPath.)
+ _ (MockedZookeeper. (proxy [Zookeeper] []
+ (zkLeaderElectorImpl [conf blob-store] (MockLeaderElector. ))))]
+ (let [nimbus-dir (.getPath tmp-nimbus-dir)]
(letlocals
(bind conf (merge (clojurify-structure (ConfigUtils/readStormConfig))
{STORM-ZOOKEEPER-SERVERS ["localhost"]
STORM-CLUSTER-MODE "local"
- STORM-ZOOKEEPER-PORT zk-port
+ STORM-ZOOKEEPER-PORT (.getPort zk)
STORM-LOCAL-DIR nimbus-dir
NIMBUS-TOPOLOGY-ACTION-NOTIFIER-PLUGIN (.getName InMemoryTopologyActionNotifier)}))
(bind cluster-state (ClusterUtils/mkStormClusterState conf nil (ClusterStateContext.)))
@@ -1573,7 +1624,7 @@
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 3))}
{}))
- (submit-local-topology nimbus "test-notification" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 30} topology)
+ (.submitTopology nimbus "test-notification" nil (str "{\"" TOPOLOGY-MESSAGE-TIMEOUT-SECS "\": 30}") topology)
(.deactivate nimbus "test-notification")
@@ -1594,23 +1645,23 @@
)))))
(deftest test-debug-on-component
- (with-local-cluster [cluster]
- (let [nimbus (:nimbus cluster)
+ (with-open [cluster (LocalCluster. )]
+ (let [nimbus (.getNimbus cluster)
topology (Thrift/buildTopology
{"spout" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 3))}
{})]
- (submit-local-topology nimbus "t1" {TOPOLOGY-WORKERS 1} topology)
+ (.submitTopology cluster "t1" {TOPOLOGY-WORKERS 1} topology)
(.debug nimbus "t1" "spout" true 100))))
(deftest test-debug-on-global
- (with-local-cluster [cluster]
- (let [nimbus (:nimbus cluster)
+ (with-open [cluster (LocalCluster. )]
+ (let [nimbus (.getNimbus cluster)
topology (Thrift/buildTopology
{"spout" (Thrift/prepareSpoutDetails
(TestPlannerSpout. true) (Integer. 3))}
{})]
- (submit-local-topology nimbus "t1" {TOPOLOGY-WORKERS 1} topology)
+ (.submitTopology cluster "t1" {TOPOLOGY-WORKERS 1} topology)
(.debug nimbus "t1" "" true 100))))
;; if the user sends an empty log config, nimbus will say that all
@@ -1618,9 +1669,12 @@
(deftest empty-save-config-results-in-all-unchanged-actions
(let [cluster-state (Mockito/mock IStormClusterState)
blob-store (Mockito/mock BlobStore)]
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store
- :daemon-conf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"}]
- (let [nimbus (:nimbus cluster)
+ (with-open [cluster (.build
+ (doto (LocalCluster$Builder. )
+ (.withClusterState cluster-state)
+ (.withBlobStore blob-store)
+ (.withDaemonConf {NIMBUS-AUTHORIZER "org.apache.storm.security.auth.authorizer.NoopAuthorizer"})))]
+ (let [nimbus (.getNimbus cluster)
previous-config (LogConfig.)
mock-config (LogConfig.)
expected-config (LogConfig.)]
@@ -1644,9 +1698,12 @@
(deftest log-level-update-merges-and-flags-existent-log-level
(let [cluster-state (Mockito/mock IStormClusterState)
blob-store (Mockito/mock BlobStore)]
- (with-mocked-nimbus [cluster :cluster-state cluster-state :blob-store blob-store
-
<TRUNCATED>
[04/10] storm git commit: STORM-1281: LocalCluster,
testing4j and testing.clj to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/CompleteTopologyParam.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/CompleteTopologyParam.java b/storm-core/src/jvm/org/apache/storm/testing/CompleteTopologyParam.java
index c124d58..00d0e57 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/CompleteTopologyParam.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/CompleteTopologyParam.java
@@ -17,7 +17,10 @@
*/
package org.apache.storm.testing;
+import java.util.Map;
+
import org.apache.storm.Config;
+import org.apache.storm.Testing;
/**
* The param class for the `Testing.completeTopology`.
@@ -26,15 +29,15 @@ public class CompleteTopologyParam {
/**
* The mocked spout sources
*/
- private MockedSources mockedSources;
+ private MockedSources mockedSources = new MockedSources();
/**
* the config for the topology when it was submitted to the cluster
*/
- private Config stormConf;
+ private Map<String, Object> stormConf = new Config();
/**
* whether cleanup the state?
*/
- private Boolean cleanupState;
+ private boolean cleanupState = true;
/**
* the topology name you want to submit to the cluster
*/
@@ -43,29 +46,39 @@ public class CompleteTopologyParam {
/**
* the timeout of topology you want to submit to the cluster
*/
- private Integer timeoutMs;
+ private int timeoutMs = Testing.TEST_TIMEOUT_MS;
public MockedSources getMockedSources() {
return mockedSources;
}
public void setMockedSources(MockedSources mockedSources) {
+ if (mockedSources == null) {
+ mockedSources = new MockedSources();
+ }
+
this.mockedSources = mockedSources;
}
- public Config getStormConf() {
+ public Map<String, Object> getStormConf() {
return stormConf;
}
- public void setStormConf(Config stormConf) {
+ public void setStormConf(Map<String, Object> stormConf) {
+ if (stormConf == null) {
+ stormConf = new Config();
+ }
this.stormConf = stormConf;
}
- public Boolean getCleanupState() {
+ public boolean getCleanupState() {
return cleanupState;
}
public void setCleanupState(Boolean cleanupState) {
+ if (cleanupState == null) {
+ cleanupState = true;
+ }
this.cleanupState = cleanupState;
}
@@ -82,6 +95,9 @@ public class CompleteTopologyParam {
}
public void setTimeoutMs(Integer timeoutMs) {
+ if (timeoutMs == null) {
+ timeoutMs = Testing.TEST_TIMEOUT_MS;
+ }
this.timeoutMs = timeoutMs;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java b/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
index 02872d0..7801ca0 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
@@ -37,6 +37,10 @@ public class FeederSpout extends BaseRichSpout {
private SpoutOutputCollector _collector;
private AckFailDelegate _ackFailDelegate;
+ public FeederSpout(List<String> outFields) {
+ this(new Fields(outFields));
+ }
+
public FeederSpout(Fields outFields) {
_id = InprocMessaging.acquireNewPort();
_outFields = outFields;
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java b/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java
index 87fc52d..feac43e 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java
@@ -30,7 +30,7 @@ import java.util.Map;
import java.util.UUID;
import static org.apache.storm.utils.Utils.get;
-public class FixedTupleSpout implements IRichSpout {
+public class FixedTupleSpout implements IRichSpout, CompletableSpout {
private static final Map<String, Integer> acked = new HashMap<String, Integer>();
private static final Map<String, Integer> failed = new HashMap<String, Integer>();
@@ -176,4 +176,9 @@ public class FixedTupleSpout implements IRichSpout {
public Map<String, Object> getComponentConfiguration() {
return null;
}
+
+ @Override
+ public boolean isExhausted() {
+ return getSourceTuples().size() == getCompleted();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/InProcessZookeeper.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/InProcessZookeeper.java b/storm-core/src/jvm/org/apache/storm/testing/InProcessZookeeper.java
new file mode 100644
index 0000000..dafde30
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/testing/InProcessZookeeper.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.testing;
+
+import java.util.List;
+
+import org.apache.storm.zookeeper.Zookeeper;
+import org.apache.zookeeper.server.NIOServerCnxnFactory;
+
+/**
+ * A local Zookeeper instance available for testing.
+ * try (InProcessZookeeper zk = new InProcessZookeeper) {
+ * // Run Tests...
+ * }
+ */
+public class InProcessZookeeper implements AutoCloseable {
+
+ private final TmpPath zkTmp;
+ private final NIOServerCnxnFactory zookeeper;
+ private final long zkPort;
+
+ public InProcessZookeeper() throws Exception {
+ zkTmp = new TmpPath();
+ @SuppressWarnings("unchecked")
+ List<Object> portAndHandle = Zookeeper.mkInprocessZookeeper(zkTmp.getPath(), null);
+ zkPort = (Long) portAndHandle.get(0);
+ zookeeper = (NIOServerCnxnFactory) portAndHandle.get(1);
+ }
+
+ /**
+ * @return the port ZK is listening on (localhost)
+ */
+ public long getPort() {
+ return zkPort;
+ }
+
+ @Override
+ public void close() throws Exception {
+ Zookeeper.shutdownInprocessZookeeper(zookeeper);
+ zkTmp.close();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/MkClusterParam.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/MkClusterParam.java b/storm-core/src/jvm/org/apache/storm/testing/MkClusterParam.java
index c3946fa..7e1e206 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/MkClusterParam.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/MkClusterParam.java
@@ -34,7 +34,7 @@ public class MkClusterParam {
/**
* cluster config
*/
- private Map daemonConf;
+ private Map<String, Object> daemonConf;
private Boolean nimbusDaemon;
@@ -53,10 +53,10 @@ public class MkClusterParam {
public void setPortsPerSupervisor(Integer portsPerSupervisor) {
this.portsPerSupervisor = portsPerSupervisor;
}
- public Map getDaemonConf() {
+ public Map<String, Object> getDaemonConf() {
return daemonConf;
}
- public void setDaemonConf(Map daemonConf) {
+ public void setDaemonConf(Map<String, Object> daemonConf) {
this.daemonConf = daemonConf;
}
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/MkTupleParam.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/MkTupleParam.java b/storm-core/src/jvm/org/apache/storm/testing/MkTupleParam.java
index ac3e1d3..93aba72 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/MkTupleParam.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/MkTupleParam.java
@@ -18,6 +18,7 @@
package org.apache.storm.testing;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
public class MkTupleParam {
@@ -42,10 +43,16 @@ public class MkTupleParam {
public List<String> getFields() {
return fields;
}
+
+ public void setFieldsList(List<String> fields) {
+ if (fields != null) {
+ this.fields = new ArrayList<>(fields);
+ } else {
+ this.fields = null;
+ }
+ }
+
public void setFields(String... fields) {
- this.fields = new ArrayList<String>();
- for (int i = 0; i < fields.length; i++) {
- this.fields.add(fields[i]);
- }
+ this.fields = new ArrayList<>(Arrays.asList(fields));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java b/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java
new file mode 100644
index 0000000..ade29a9
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.testing;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.storm.nimbus.ILeaderElector;
+import org.apache.storm.nimbus.NimbusInfo;
+
+public class MockLeaderElector implements ILeaderElector {
+ private final boolean isLeader;
+ private final NimbusInfo leaderAddress;
+
+ public MockLeaderElector() {
+ this(true, "test-host", 9999);
+ }
+
+ public MockLeaderElector(boolean isLeader) {
+ this(isLeader, "test-host", 9999);
+ }
+
+ public MockLeaderElector(boolean isLeader, String host, int port) {
+ this.isLeader = isLeader;
+ this.leaderAddress = new NimbusInfo(host, port, true);
+ }
+
+ @Override
+ public void prepare(Map conf) {
+ //NOOP
+ }
+
+ @Override
+ public void addToLeaderLockQueue() throws Exception {
+ //NOOP
+ }
+
+ @Override
+ public void removeFromLeaderLockQueue() throws Exception {
+ //NOOP
+ }
+
+ @Override
+ public boolean isLeader() throws Exception {
+ return isLeader;
+ }
+
+ @Override
+ public NimbusInfo getLeader() {
+ return leaderAddress;
+ }
+
+ @Override
+ public List<NimbusInfo> getAllNimbuses() throws Exception {
+ return Arrays.asList(leaderAddress);
+ }
+
+ @Override
+ public void close() {
+ //NOOP
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/MockedSources.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/MockedSources.java b/storm-core/src/jvm/org/apache/storm/testing/MockedSources.java
index 1063ad2..61df4b6 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/MockedSources.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/MockedSources.java
@@ -31,6 +31,14 @@ public class MockedSources {
*/
private Map<String, List<FixedTuple>> data = new HashMap<String, List<FixedTuple>>();
+ public MockedSources() {
+ //Empty
+ }
+
+ public MockedSources(Map<String, List<FixedTuple>> data) {
+ this.data = new HashMap<>(data);
+ }
+
/**
* add mock data for the spout.
*
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/TestEventLogSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/TestEventLogSpout.java b/storm-core/src/jvm/org/apache/storm/testing/TestEventLogSpout.java
index 0e04a6a..3f310e3 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/TestEventLogSpout.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/TestEventLogSpout.java
@@ -34,7 +34,7 @@ import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
-public class TestEventLogSpout extends BaseRichSpout {
+public class TestEventLogSpout extends BaseRichSpout implements CompletableSpout {
public static Logger LOG = LoggerFactory.getLogger(TestEventLogSpout.class);
private static final Map<String, Integer> acked = new HashMap<String, Integer>();
@@ -136,4 +136,9 @@ public class TestEventLogSpout extends BaseRichSpout {
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("source", "eventId"));
}
+
+ @Override
+ public boolean isExhausted() {
+ return completed();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/TmpPath.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/TmpPath.java b/storm-core/src/jvm/org/apache/storm/testing/TmpPath.java
new file mode 100644
index 0000000..ccb1b2f
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/testing/TmpPath.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.testing;
+
+import java.io.File;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TmpPath implements AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(TmpPath.class);
+
+ public static String localTempPath() {
+ StringBuilder ret = new StringBuilder().append(System.getProperty("java.io.tmpdir"));
+ if (!Utils.isOnWindows()) {
+ ret.append("/");
+ }
+ ret.append(Utils.uuid());
+ return ret.toString();
+ }
+
+ private final File path;
+
+ public TmpPath() {
+ this(localTempPath());
+ }
+
+ public TmpPath(String path) {
+ this.path = new File(path);
+ }
+
+ public String getPath() {
+ return path.getAbsolutePath();
+ }
+
+ public File getFile() {
+ return path;
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (path.exists()) {
+ try {
+ FileUtils.forceDelete(path);
+ } catch (Exception e) {
+ //on windows, the host process still holds lock on the logfile
+ LOG.info(e.getMessage());
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/TrackedTopology.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/TrackedTopology.java b/storm-core/src/jvm/org/apache/storm/testing/TrackedTopology.java
index ff3a32c..00bd1e7 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/TrackedTopology.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/TrackedTopology.java
@@ -17,18 +17,119 @@
*/
package org.apache.storm.testing;
-import java.util.HashMap;
-import java.util.Map;
+import static org.apache.storm.Testing.whileTimeout;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.storm.ILocalCluster;
+import org.apache.storm.Testing;
+import org.apache.storm.Thrift;
+import org.apache.storm.generated.Bolt;
+import org.apache.storm.generated.SpoutSpec;
import org.apache.storm.generated.StormTopology;
-import clojure.lang.Keyword;
+import org.apache.storm.topology.IRichBolt;
+import org.apache.storm.topology.IRichSpout;
+import org.apache.storm.utils.RegisteredGlobalState;
+import org.apache.storm.utils.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class TrackedTopology extends HashMap{
- public TrackedTopology(Map map) {
- super(map);
+/**
+ * A tracked topology keeps metrics for every bolt and spout.
+ * This allows a test to know how many tuples have been fully processed.
+ * Metrics are tracked on a per cluster basis. So only one tracked topology
+ * should be run in a tracked cluster to avoid conflicts.
+ */
+public class TrackedTopology {
+ private static final Logger LOG = LoggerFactory.getLogger(TrackedTopology.class);
+ private final StormTopology topology;
+ private final AtomicInteger lastSpoutCommit;
+ private final ILocalCluster cluster;
+
+ /**
+ * Create a new topology to be tracked.
+ * @param origTopo the original topology.
+ * @param cluster a cluster that should have been launched with tracking enabled.
+ */
+ public TrackedTopology(StormTopology origTopo, ILocalCluster cluster) {
+ LOG.warn("CLUSTER {} - {}", cluster, cluster.getTrackedId());
+ this.cluster = cluster;
+ lastSpoutCommit = new AtomicInteger(0);
+ String id = cluster.getTrackedId();
+ topology = origTopo.deepCopy();
+ for (Bolt bolt: topology.get_bolts().values()) {
+ IRichBolt obj = (IRichBolt) Thrift.deserializeComponentObject(bolt.get_bolt_object());
+ bolt.set_bolt_object(Thrift.serializeComponentObject(new BoltTracker(obj, id)));
+ }
+ for (SpoutSpec spout: topology.get_spouts().values()) {
+ IRichSpout obj = (IRichSpout) Thrift.deserializeComponentObject(spout.get_spout_object());
+ spout.set_spout_object(Thrift.serializeComponentObject(new SpoutTracker(obj, id)));
+ }
}
+
+ public StormTopology getTopology() {
+ return topology;
+ }
- public StormTopology getTopology() {
- return (StormTopology)get(Keyword.intern("topology"));
+ public ILocalCluster getCluster() {
+ return cluster;
}
+
+ /**
+ * Wait for 1 tuple to be fully processed
+ */
+ public void trackedWait() {
+ trackedWait(1, Testing.TEST_TIMEOUT_MS);
+ }
+
+ /**
+ * Wait for amt tuples to be fully processed.
+ */
+ public void trackedWait(int amt) {
+ trackedWait(amt, Testing.TEST_TIMEOUT_MS);
+ }
+
+ /**
+ * Wait for amt tuples to be fully processed timeoutMs happens.
+ */
+ public void trackedWait(int amt, int timeoutMs) {
+ final int target = amt + lastSpoutCommit.get();
+ final String id = cluster.getTrackedId();
+ Random rand = ThreadLocalRandom.current();
+ whileTimeout(timeoutMs,
+ () -> {
+ int se = globalAmt(id, "spout-emitted");
+ int transferred = globalAmt(id, "transferred");
+ int processed = globalAmt(id, "processed");
+ LOG.info("emitted {} target {} transferred {} processed {}", se, target, transferred, processed);
+ return (target != se) || (transferred != processed);
+ },
+ () -> {
+ Time.advanceTimeSecs(1);
+ try {
+ Thread.sleep(rand.nextInt(200));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ lastSpoutCommit.set(target);
+ }
+
+ /**
+ * Read a metric from the tracked cluster (NOT JUST THIS TOPOLOGY)
+ * @param key one of "spout-emitted", "processed", or "transferred"
+ * @return the amount of that metric
+ */
+ public int globalAmt(String key) {
+ return globalAmt(cluster.getTrackedId(), key);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static int globalAmt(String id, String key) {
+ LOG.warn("Reading tracked metrics for ID {}", id);
+ return ((ConcurrentHashMap<String, AtomicInteger>)RegisteredGlobalState.getState(id)).get(key).get();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/topology/TopologyBuilder.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/TopologyBuilder.java b/storm-core/src/jvm/org/apache/storm/topology/TopologyBuilder.java
index 40ede4c..a86036e 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/TopologyBuilder.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/TopologyBuilder.java
@@ -88,10 +88,10 @@ import static org.apache.storm.spout.CheckpointSpout.CHECKPOINT_STREAM_ID;
* conf.put(Config.TOPOLOGY_WORKERS, 4);
* conf.put(Config.TOPOLOGY_DEBUG, true);
*
- * LocalCluster cluster = new LocalCluster();
- * cluster.submitTopology("mytopology", conf, builder.createTopology());
- * Utils.sleep(10000);
- * cluster.shutdown();
+ * try (LocalCluster cluster = new LocalCluster();
+ * LocalTopology topo = cluster.submitTopology("mytopology", conf, builder.createTopology());){
+ * Utils.sleep(10000);
+ * }
* ```
*
* The pattern for `TopologyBuilder` is to map component ids to components using the setSpout
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/utils/Time.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Time.java b/storm-core/src/jvm/org/apache/storm/utils/Time.java
index 9b656ee..b6c48c1 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Time.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Time.java
@@ -26,36 +26,55 @@ import org.slf4j.LoggerFactory;
public class Time {
- public static final Logger LOG = LoggerFactory.getLogger(Time.class);
-
+ private static final Logger LOG = LoggerFactory.getLogger(Time.class);
private static AtomicBoolean simulating = new AtomicBoolean(false);
private static AtomicLong autoAdvanceOnSleep = new AtomicLong(0);
- //TODO: should probably use weak references here or something
private static volatile Map<Thread, AtomicLong> threadSleepTimes;
private static final Object sleepTimesLock = new Object();
+ private static AtomicLong simulatedCurrTimeMs;
- private static AtomicLong simulatedCurrTimeMs; //should this be a thread local that's allowed to keep advancing?
-
- public static void startSimulating() {
- synchronized(sleepTimesLock) {
- simulating.set(true);
- simulatedCurrTimeMs = new AtomicLong(0);
- threadSleepTimes = new ConcurrentHashMap<>();
+ public static class SimulatedTime implements AutoCloseable {
+
+ public SimulatedTime() {
+ this(null);
+ }
+
+ public SimulatedTime(Number ms) {
+ synchronized(Time.sleepTimesLock) {
+ Time.simulating.set(true);
+ Time.simulatedCurrTimeMs = new AtomicLong(0);
+ Time.threadSleepTimes = new ConcurrentHashMap<>();
+ if (ms != null) {
+ Time.autoAdvanceOnSleep.set(ms.longValue());
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ synchronized(Time.sleepTimesLock) {
+ Time.simulating.set(false);
+ Time.autoAdvanceOnSleep.set(0);
+ Time.threadSleepTimes = null;
+ }
}
}
- public static void startSimulatingAutoAdvanceOnSleep(long ms) {
- synchronized(sleepTimesLock) {
- startSimulating();
- autoAdvanceOnSleep.set(ms);
+ @Deprecated
+ public static void startSimulating() {
+ synchronized(Time.sleepTimesLock) {
+ Time.simulating.set(true);
+ Time.simulatedCurrTimeMs = new AtomicLong(0);
+ Time.threadSleepTimes = new ConcurrentHashMap<>();
}
}
+ @Deprecated
public static void stopSimulating() {
- synchronized(sleepTimesLock) {
- simulating.set(false);
- autoAdvanceOnSleep.set(0);
- threadSleepTimes = null;
+ synchronized(Time.sleepTimesLock) {
+ Time.simulating.set(false);
+ Time.autoAdvanceOnSleep.set(0);
+ Time.threadSleepTimes = null;
}
}
@@ -145,6 +164,10 @@ public class Time {
LOG.debug("Advanced simulated time to {}", newTime);
}
+ public static void advanceTimeSecs(long secs) {
+ advanceTime(secs * 1_000);
+ }
+
public static boolean isThreadWaiting(Thread t) {
if(!simulating.get()) throw new IllegalStateException("Must be in simulation mode");
AtomicLong time;
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/integration/org/apache/storm/integration_test.clj b/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
index 925f81b..9c54aed 100644
--- a/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
+++ b/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
@@ -15,24 +15,26 @@
;; limitations under the License.
(ns integration.org.apache.storm.integration-test
(:use [clojure test])
- (:import [org.apache.storm Config Thrift])
+ (:import [org.apache.storm Testing LocalCluster$Builder Config Thrift])
(:import [org.apache.storm.topology TopologyBuilder])
(:import [org.apache.storm.generated InvalidTopologyException SubmitOptions TopologyInitialStatus RebalanceOptions])
- (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount
+ (:import [org.apache.storm.testing TrackedTopology MockedSources TestWordCounter TestWordSpout TestGlobalCount FeederSpout CompleteTopologyParam
TestAggregatesCounter TestConfBolt AckFailMapTracker AckTracker TestPlannerSpout])
(:import [org.apache.storm.utils Time])
(:import [org.apache.storm.tuple Fields])
(:import [org.apache.storm.cluster StormClusterStateImpl])
(:use [org.apache.storm.internal clojure])
- (:use [org.apache.storm testing config util])
+ (:use [org.apache.storm config util])
(:import [org.apache.storm Thrift])
(:import [org.apache.storm.utils Utils])
(:import [org.apache.storm.daemon StormCommon]))
(deftest test-basic-topology
(doseq [zmq-on? [true false]]
- (with-simulated-time-local-cluster [cluster :supervisors 4
- :daemon-conf {STORM-LOCAL-MODE-ZMQ zmq-on?}]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withSupervisors 4)
+ (.withDaemonConf {STORM-LOCAL-MODE-ZMQ zmq-on?})))]
(let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestWordSpout. true) (Integer. 3))}
@@ -48,19 +50,20 @@
{(Utils/getGlobalStreamId "2" nil)
(Thrift/prepareGlobalGrouping)}
(TestAggregatesCounter.))})
- results (complete-topology cluster
+ results (Testing/completeTopology cluster
topology
- :mock-sources {"1" [["nathan"] ["bob"] ["joey"] ["nathan"]]}
- :storm-conf {TOPOLOGY-WORKERS 2
- TOPOLOGY-TESTING-ALWAYS-TRY-SERIALIZE true})]
- (is (ms= [["nathan"] ["bob"] ["joey"] ["nathan"]]
- (read-tuples results "1")))
- (is (ms= [["nathan" 1] ["nathan" 2] ["bob" 1] ["joey" 1]]
- (read-tuples results "2")))
+ (doto (CompleteTopologyParam.)
+ (.setMockedSources (MockedSources. {"1" [["nathan"] ["bob"] ["joey"] ["nathan"]]}))
+ (.setStormConf {TOPOLOGY-WORKERS 2
+ TOPOLOGY-TESTING-ALWAYS-TRY-SERIALIZE true})))]
+ (is (Testing/multiseteq [["nathan"] ["bob"] ["joey"] ["nathan"]]
+ (Testing/readTuples results "1")))
+ (is (Testing/multiseteq [["nathan" (int 1)] ["nathan" (int 2)] ["bob" (int 1)] ["joey" (int 1)]]
+ (Testing/readTuples results "2")))
(is (= [[1] [2] [3] [4]]
- (read-tuples results "3")))
+ (Testing/readTuples results "3")))
(is (= [[1] [2] [3] [4]]
- (read-tuples results "4")))
+ (Testing/readTuples results "4")))
))))
(defbolt emit-task-id ["tid"] {:prepare true}
@@ -73,7 +76,9 @@
))))
(deftest test-multi-tasks-per-executor
- (with-simulated-time-local-cluster [cluster :supervisors 4]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withSupervisors 4)))]
(let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails (TestWordSpout. true))}
{"2" (Thrift/prepareBoltDetails
@@ -82,11 +87,12 @@
emit-task-id
(Integer. 3)
{TOPOLOGY-TASKS 6})})
- results (complete-topology cluster
+ results (Testing/completeTopology cluster
topology
- :mock-sources {"1" [["a"]]})]
- (is (ms= [[0] [1] [2] [3] [4] [5]]
- (read-tuples results "2")))
+ (doto (CompleteTopologyParam.)
+ (.setMockedSources (MockedSources. {"1" [["a"]]}))))]
+ (is (Testing/multiseteq [[(int 0)] [(int 1)] [(int 2)] [(int 3)] [(int 4)] [(int 5)]]
+ (Testing/readTuples results "2")))
)))
(defbolt ack-every-other {} {:prepare true}
@@ -119,8 +125,11 @@
(assert-loop #(.isFailed tracker %) ids))
(deftest test-timeout
- (with-simulated-time-local-cluster [cluster :daemon-conf {TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true}]
- (let [feeder (feeder-spout ["field1"])
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withSupervisors 4)
+ (.withDaemonConf {TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true})))]
+ (let [feeder (FeederSpout. ["field1"])
tracker (AckFailMapTracker.)
_ (.setAckFailDelegate feeder tracker)
topology (Thrift/buildTopology
@@ -128,18 +137,18 @@
{"2" (Thrift/prepareBoltDetails
{(Utils/getGlobalStreamId "1" nil)
(Thrift/prepareGlobalGrouping)} ack-every-other)})]
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"timeout-tester"
{TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
topology)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(.feed feeder ["a"] 1)
(.feed feeder ["b"] 2)
(.feed feeder ["c"] 3)
- (advance-cluster-time cluster 9)
+ (.advanceClusterTime cluster 9)
(assert-acked tracker 1 3)
(is (not (.isFailed tracker 2)))
- (advance-cluster-time cluster 12)
+ (.advanceClusterTime cluster 12)
(assert-failed tracker 2)
)))
@@ -158,8 +167,10 @@
)))))
(deftest test-reset-timeout
- (with-simulated-time-local-cluster [cluster :daemon-conf {TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true}]
- (let [feeder (feeder-spout ["field1"])
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true})))]
+ (let [feeder (FeederSpout. ["field1"])
tracker (AckFailMapTracker.)
_ (.setAckFailDelegate feeder tracker)
topology (Thrift/buildTopology
@@ -167,16 +178,16 @@
{"2" (Thrift/prepareBoltDetails
{(Utils/getGlobalStreamId "1" nil)
(Thrift/prepareGlobalGrouping)} extend-timeout-twice)})]
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"timeout-tester"
{TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
topology)
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(.feed feeder ["a"] 1)
- (advance-cluster-time cluster 21)
+ (.advanceClusterTime cluster 21)
(is (not (.isFailed tracker 1)))
(is (not (.isAcked tracker 1)))
- (advance-cluster-time cluster 5)
+ (.advanceClusterTime cluster 5)
(assert-acked tracker 1)
)))
@@ -214,15 +225,18 @@
(defn try-complete-wc-topology [cluster topology]
(try (do
- (complete-topology cluster
+ (Testing/completeTopology cluster
topology
- :mock-sources {"1" [["nathan"] ["bob"] ["joey"] ["nathan"]]}
- :storm-conf {TOPOLOGY-WORKERS 2})
+ (doto (CompleteTopologyParam.)
+ (.setMockedSources (MockedSources. {"1" [["nathan"] ["bob"] ["joey"] ["nathan"]]}))
+ (.setStormConf {TOPOLOGY-WORKERS 2})))
false)
(catch InvalidTopologyException e true)))
(deftest test-validate-topology-structure
- (with-simulated-time-local-cluster [cluster :supervisors 4]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSupervisors 4)
+ (.withSimulatedTime)))]
(let [any-error1? (try-complete-wc-topology cluster (mk-validate-topology-1))
any-error2? (try-complete-wc-topology cluster (mk-invalidate-topology-1))
any-error3? (try-complete-wc-topology cluster (mk-invalidate-topology-2))
@@ -239,7 +253,8 @@
(deftest test-system-stream
;; this test works because mocking a spout splits up the tuples evenly among the tasks
- (with-simulated-time-local-cluster [cluster]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)))]
(let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestWordSpout. true) (Integer. 3))}
@@ -249,17 +264,18 @@
(Utils/getGlobalStreamId "1" "__system")
(Thrift/prepareGlobalGrouping)}
identity-bolt (Integer. 1))})
- results (complete-topology cluster
+ results (Testing/completeTopology cluster
topology
- :mock-sources {"1" [["a"] ["b"] ["c"]]}
- :storm-conf {TOPOLOGY-WORKERS 2})]
- (is (ms= [["a"] ["b"] ["c"] ["startup"] ["startup"] ["startup"]]
- (read-tuples results "2")))
+ (doto (CompleteTopologyParam.)
+ (.setMockedSources (MockedSources. {"1" [["a"] ["b"] ["c"]]}))
+ (.setStormConf {TOPOLOGY-WORKERS 2})))]
+ (is (Testing/multiseteq [["a"] ["b"] ["c"] ["startup"] ["startup"] ["startup"]]
+ (Testing/readTuples results "2")))
)))
(defn ack-tracking-feeder [fields]
(let [tracker (AckTracker.)]
- [(doto (feeder-spout fields)
+ [(doto (FeederSpout. fields)
(.setAckFailDelegate tracker))
(fn [val]
(is (= (.getNumAcks tracker) val))
@@ -293,12 +309,11 @@
(ack! collector tuple))
(deftest test-acking
- (with-tracked-cluster [cluster]
+ (with-open [cluster (.build (.withSimulatedTime (.withTracked (LocalCluster$Builder. ))))]
(let [[feeder1 checker1] (ack-tracking-feeder ["num"])
[feeder2 checker2] (ack-tracking-feeder ["num"])
[feeder3 checker3] (ack-tracking-feeder ["num"])
- tracked (mk-tracked-topology
- cluster
+ tracked (TrackedTopology.
(Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails feeder1)
"2" (Thrift/prepareSpoutDetails feeder2)
@@ -331,42 +346,40 @@
{(Utils/getGlobalStreamId "8" nil)
(Thrift/prepareShuffleGrouping)}
ack-bolt)}
- ))]
- (submit-local-topology (:nimbus cluster)
- "acking-test1"
- {}
- (:topology tracked))
- (advance-cluster-time cluster 11)
+ )
+ cluster)]
+ (.submitTopology cluster
+ "acking-test1"
+ {}
+ tracked)
+ (.advanceClusterTime cluster 11)
(.feed feeder1 [1])
- (tracked-wait tracked 1)
+ (Testing/trackedWait tracked (int 1))
(checker1 0)
(.feed feeder2 [1])
- (tracked-wait tracked 1)
+ (Testing/trackedWait tracked (int 1))
(checker1 1)
(checker2 1)
(.feed feeder1 [1])
- (tracked-wait tracked 1)
+ (Testing/trackedWait tracked (int 1))
(checker1 0)
(.feed feeder1 [1])
- (tracked-wait tracked 1)
+ (Testing/trackedWait tracked (int 1))
(checker1 1)
(.feed feeder3 [1])
- (tracked-wait tracked 1)
+ (Testing/trackedWait tracked (int 1))
(checker1 0)
(checker3 0)
(.feed feeder2 [1])
- (tracked-wait tracked 1)
+ (Testing/trackedWait tracked (int 1))
(checker1 1)
(checker2 1)
- (checker3 1)
-
- )))
+ (checker3 1))))
(deftest test-ack-branching
- (with-tracked-cluster [cluster]
+ (with-open [cluster (.build (.withSimulatedTime (.withTracked (LocalCluster$Builder. ))))]
(let [[feeder checker] (ack-tracking-feeder ["num"])
- tracked (mk-tracked-topology
- cluster
+ tracked (TrackedTopology.
(Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails feeder)}
{"2" (Thrift/prepareBoltDetails
@@ -382,19 +395,19 @@
(Thrift/prepareShuffleGrouping)
(Utils/getGlobalStreamId "3" nil)
(Thrift/prepareShuffleGrouping)}
- (agg-bolt 4))}))]
- (submit-local-topology (:nimbus cluster)
- "test-acking2"
- {}
- (:topology tracked))
- (advance-cluster-time cluster 11)
+ (agg-bolt 4))})
+ cluster)]
+ (.submitTopology cluster
+ "test-acking2"
+ {}
+ tracked)
+ (.advanceClusterTime cluster 11)
(.feed feeder [1])
- (tracked-wait tracked 1)
+ (Testing/trackedWait tracked (int 1))
(checker 0)
(.feed feeder [1])
- (tracked-wait tracked 1)
- (checker 2)
- )))
+ (Testing/trackedWait tracked (int 1))
+ (checker 2))))
(defbolt dup-anchor ["num"]
[tuple collector]
@@ -418,8 +431,10 @@
)))
(deftest test-submit-inactive-topology
- (with-simulated-time-local-cluster [cluster :daemon-conf {TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true}]
- (let [feeder (feeder-spout ["field1"])
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true})))]
+ (let [feeder (FeederSpout. ["field1"])
tracker (AckFailMapTracker.)
_ (.setAckFailDelegate feeder tracker)
topology (Thrift/buildTopology
@@ -432,28 +447,27 @@
(reset! bolt-prepared? false)
(reset! spout-opened? false)
- (submit-local-topology-with-opts (:nimbus cluster)
+ (.submitTopologyWithOpts cluster
"test"
{TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
topology
(SubmitOptions. TopologyInitialStatus/INACTIVE))
- (advance-cluster-time cluster 11)
+ (.advanceClusterTime cluster 11)
(.feed feeder ["a"] 1)
- (advance-cluster-time cluster 9)
+ (.advanceClusterTime cluster 9)
(is (not @bolt-prepared?))
(is (not @spout-opened?))
- (.activate (:nimbus cluster) "test")
+ (.activate (.getNimbus cluster) "test")
- (advance-cluster-time cluster 12)
+ (.advanceClusterTime cluster 12)
(assert-acked tracker 1)
(is @bolt-prepared?)
(is @spout-opened?))))
(deftest test-acking-self-anchor
- (with-tracked-cluster [cluster]
+ (with-open [cluster (.build (.withSimulatedTime (.withTracked (LocalCluster$Builder. ))))]
(let [[feeder checker] (ack-tracking-feeder ["num"])
- tracked (mk-tracked-topology
- cluster
+ tracked (TrackedTopology.
(Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails feeder)}
{"2" (Thrift/prepareBoltDetails
@@ -463,21 +477,21 @@
"3" (Thrift/prepareBoltDetails
{(Utils/getGlobalStreamId "2" nil)
(Thrift/prepareShuffleGrouping)}
- ack-bolt)}))]
- (submit-local-topology (:nimbus cluster)
- "test"
- {}
- (:topology tracked))
- (advance-cluster-time cluster 11)
+ ack-bolt)})
+ cluster)]
+ (.submitTopology cluster
+ "test"
+ {}
+ tracked)
+ (.advanceClusterTime cluster 11)
(.feed feeder [1])
- (tracked-wait tracked 1)
+ (Testing/trackedWait tracked (int 1))
(checker 1)
(.feed feeder [1])
(.feed feeder [1])
(.feed feeder [1])
- (tracked-wait tracked 3)
- (checker 3)
- )))
+ (Testing/trackedWait tracked (int 3))
+ (checker 3))))
(defspout IncSpout ["word"]
[conf context collector]
@@ -504,22 +518,25 @@
)))
(deftest test-kryo-decorators-config
- (with-simulated-time-local-cluster [cluster
- :daemon-conf {TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true
- TOPOLOGY-KRYO-DECORATORS ["this-is-overriden"]}]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true
+ TOPOLOGY-KRYO-DECORATORS ["this-is-overriden"]})))]
(let [builder (TopologyBuilder.)
_ (.setSpout builder "1" (TestPlannerSpout. (Fields. ["conf"])))
_ (-> builder (.setBolt "2" (TestConfBolt. {TOPOLOGY-KRYO-DECORATORS ["one" "two"]})) (.shuffleGrouping "1"))
- results (complete-topology cluster
+ results (Testing/completeTopology cluster
(.createTopology builder)
- :storm-conf {TOPOLOGY-KRYO-DECORATORS ["one" "three"]}
- :mock-sources {"1" [[TOPOLOGY-KRYO-DECORATORS]]})]
+ (doto (CompleteTopologyParam.)
+ (.setStormConf {TOPOLOGY-KRYO-DECORATORS ["one" "three"]})
+ (.setMockedSources (MockedSources. {"1" [[TOPOLOGY-KRYO-DECORATORS]]}))))]
(is (= {"topology.kryo.decorators" (list "one" "two" "three")}
- (->> (read-tuples results "2") (apply concat) (apply hash-map)))))))
+ (->> (Testing/readTuples results "2") (apply concat) (apply hash-map)))))))
(deftest test-component-specific-config
- (with-simulated-time-local-cluster [cluster
- :daemon-conf {TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true}]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true})))]
(let [builder (TopologyBuilder.)
_ (.setSpout builder "1" (TestPlannerSpout. (Fields. ["conf"])))
_ (-> builder
@@ -533,14 +550,15 @@
(.shuffleGrouping "1")
(.setMaxTaskParallelism (int 2))
(.addConfiguration "fake.config2" 987))
- results (complete-topology cluster
+ results (Testing/completeTopology cluster
(.createTopology builder)
- :storm-conf {TOPOLOGY-KRYO-REGISTER [{"fake.type" "good.serializer", "fake.type3" "a.serializer3"}]}
- :mock-sources {"1" [["fake.config"]
+ (doto (CompleteTopologyParam.)
+ (.setStormConf {TOPOLOGY-KRYO-REGISTER [{"fake.type" "good.serializer", "fake.type3" "a.serializer3"}]})
+ (.setMockedSources (MockedSources. {"1" [["fake.config"]
[TOPOLOGY-MAX-TASK-PARALLELISM]
[TOPOLOGY-MAX-SPOUT-PENDING]
["fake.config2"]
- [TOPOLOGY-KRYO-REGISTER]]})]
+ [TOPOLOGY-KRYO-REGISTER]]}))))]
(is (= {"fake.config" 123
"fake.config2" 987
TOPOLOGY-MAX-TASK-PARALLELISM 2
@@ -548,7 +566,7 @@
TOPOLOGY-KRYO-REGISTER {"fake.type" "good.serializer"
"fake.type2" "a.serializer"
"fake.type3" "a.serializer3"}}
- (->> (read-tuples results "2")
+ (->> (Testing/readTuples results "2")
(apply concat)
(apply hash-map)))))))
@@ -582,7 +600,8 @@
))))
(deftest test-hooks
- (with-simulated-time-local-cluster [cluster]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)))]
(let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestPlannerSpout. (Fields. ["conf"])))}
@@ -590,18 +609,19 @@
{(Utils/getGlobalStreamId "1" nil)
(Thrift/prepareShuffleGrouping)}
hooks-bolt)})
- results (complete-topology cluster
+ results (Testing/completeTopology cluster
topology
- :mock-sources {"1" [[1]
+ (doto (CompleteTopologyParam.)
+ (.setMockedSources (MockedSources. {"1" [[1]
[1]
[1]
[1]
- ]})]
+ ]}))))]
(is (= [[0 0 0 0]
[2 1 0 1]
[4 1 1 2]
[6 2 1 3]]
- (read-tuples results "2")
+ (Testing/readTuples results "2")
)))))
(defbolt report-errors-bolt {}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj b/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
index 342bba6..5b437c7 100644
--- a/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
+++ b/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
@@ -15,7 +15,7 @@
;; limitations under the License.
(ns integration.org.apache.storm.testing4j-test
(:use [clojure.test])
- (:use [org.apache.storm config testing util])
+ (:use [org.apache.storm config util])
(:use [org.apache.storm.internal clojure])
(:require [integration.org.apache.storm.integration-test :as it])
(:require [org.apache.storm.internal.thrift :as thrift])
@@ -23,7 +23,7 @@
[org.apache.storm.generated GlobalStreamId])
(:import [org.apache.storm.tuple Values Tuple])
(:import [org.apache.storm.utils Time Utils])
- (:import [org.apache.storm.testing MkClusterParam TestJob MockedSources TestWordSpout
+ (:import [org.apache.storm.testing MkClusterParam TestJob MockedSources TestWordSpout FeederSpout
TestWordCounter TestGlobalCount TestAggregatesCounter CompleteTopologyParam
AckFailMapTracker MkTupleParam])
(:import [org.apache.storm.utils Utils])
@@ -45,8 +45,7 @@
(Testing/withLocalCluster mk-cluster-param (reify TestJob
(^void run [this ^ILocalCluster cluster]
(is (not (nil? cluster)))
- (is (not (nil? (.getState cluster))))
- (is (not (nil? (:nimbus (.getState cluster))))))))))
+ (is (not (nil? (.getNimbus cluster)))))))))
(deftest test-with-simulated-time-local-cluster
(let [mk-cluster-param (doto (MkClusterParam.)
@@ -58,8 +57,7 @@
(Testing/withSimulatedTimeLocalCluster mk-cluster-param (reify TestJob
(^void run [this ^ILocalCluster cluster]
(is (not (nil? cluster)))
- (is (not (nil? (.getState cluster))))
- (is (not (nil? (:nimbus (.getState cluster)))))
+ (is (not (nil? (.getNimbus cluster))))
(is (Time/isSimulating)))))
(is (not (Time/isSimulating)))))
@@ -96,8 +94,8 @@
complete-topology-param)]
(is (Testing/multiseteq [["nathan"] ["bob"] ["joey"] ["nathan"]]
(Testing/readTuples results "1")))
- (is (Testing/multiseteq [["nathan" 1] ["nathan" 2] ["bob" 1] ["joey" 1]]
- (read-tuples results "2")))
+ (is (Testing/multiseteq [["nathan" (int 1)] ["nathan" (int 2)] ["bob" (int 1)] ["joey" (int 1)]]
+ (Testing/readTuples results "2")))
(is (= [[1] [2] [3] [4]]
(Testing/readTuples results "3")))
(is (= [[1] [2] [3] [4]]
@@ -143,7 +141,7 @@
"test-acking2"
(Config.)
(.getTopology tracked))
- (advance-cluster-time (.getState cluster) 11)
+ (.advanceClusterTime cluster (int 11))
(.feed feeder [1])
(Testing/trackedWait tracked (int 1))
(checker 0)
@@ -161,7 +159,7 @@
mk-cluster-param
(reify TestJob
(^void run [this ^ILocalCluster cluster]
- (let [feeder (feeder-spout ["field1"])
+ (let [feeder (FeederSpout. ["field1"])
tracker (AckFailMapTracker.)
_ (.setAckFailDelegate feeder tracker)
topology (Thrift/buildTopology
@@ -195,7 +193,7 @@
mk-cluster-param
(reify TestJob
(^void run [this ^ILocalCluster cluster]
- (let [feeder (feeder-spout ["field1"])
+ (let [feeder (FeederSpout. ["field1"])
tracker (AckFailMapTracker.)
_ (.setAckFailDelegate feeder tracker)
topology (Thrift/buildTopology
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/integration/org/apache/storm/trident/integration_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/integration/org/apache/storm/trident/integration_test.clj b/storm-core/test/clj/integration/org/apache/storm/trident/integration_test.clj
index c55903e..d55009b 100644
--- a/storm-core/test/clj/integration/org/apache/storm/trident/integration_test.clj
+++ b/storm-core/test/clj/integration/org/apache/storm/trident/integration_test.clj
@@ -15,18 +15,23 @@
;; limitations under the License.
(ns integration.org.apache.storm.trident.integration-test
(:use [clojure test])
- (:require [org.apache.storm [testing :as t]])
+ (:import [org.apache.storm Testing LocalCluster$Builder LocalCluster LocalDRPC])
(:import [org.apache.storm.trident.testing Split CountAsAggregator StringLength TrueFilter
- MemoryMapState$Factory])
+ MemoryMapState$Factory FeederCommitterBatchSpout FeederBatchSpout])
(:import [org.apache.storm.trident.state StateSpec])
- (:import [org.apache.storm.trident.operation.impl CombinerAggStateUpdater]
+ (:import [org.apache.storm.trident TridentTopology]
+ [org.apache.storm.trident.operation.impl CombinerAggStateUpdater]
[org.apache.storm.trident.operation BaseFunction]
+ [org.apache.storm.trident.operation.builtin Count Sum Equals MapGet Debug FilterNull FirstN TupleCollectionGet]
+ [org.apache.storm.tuple Fields]
[org.json.simple.parser JSONParser]
+ [org.json.simple JSONValue]
[org.apache.storm Config])
- (:use [org.apache.storm.trident testing]
- [org.apache.storm log util config]))
+ (:use [org.apache.storm log util config]))
-(bootstrap-imports)
+(defn exec-drpc [^LocalDRPC drpc function-name args]
+ (if-let [res (.execute drpc function-name args)]
+ (clojurify-structure (JSONValue/parse res))))
(defmacro letlocals
[& body]
@@ -41,58 +46,58 @@
~(first lexpr))))
(deftest test-memory-map-get-tuples
- (t/with-local-cluster [cluster]
- (with-drpc [drpc]
+ (with-open [cluster (LocalCluster. )]
+ (with-open [drpc (LocalDRPC.)]
(letlocals
(bind topo (TridentTopology.))
- (bind feeder (feeder-spout ["sentence"]))
+ (bind feeder (FeederBatchSpout. ["sentence"]))
(bind word-counts
(-> topo
(.newStream "tester" feeder)
- (.each (fields "sentence") (Split.) (fields "word"))
- (.groupBy (fields "word"))
- (.persistentAggregate (memory-map-state) (Count.) (fields "count"))
+ (.each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
+ (.persistentAggregate (MemoryMapState$Factory.) (Count.) (Fields. ["count"]))
(.parallelismHint 6)
))
(-> topo
(.newDRPCStream "all-tuples" drpc)
(.broadcast)
- (.stateQuery word-counts (fields "args") (TupleCollectionGet.) (fields "word" "count"))
- (.project (fields "word" "count")))
- (with-topology [cluster topo storm-topo]
- (feed feeder [["hello the man said"] ["the"]])
+ (.stateQuery word-counts (Fields. ["args"]) (TupleCollectionGet.) (Fields. ["word" "count"]))
+ (.project (Fields. ["word" "count"])))
+ (with-open [storm-topo (.submitTopology cluster "testing" {} (.build topo))]
+ (.feed feeder [["hello the man said"] ["the"]])
(is (= #{["hello" 1] ["said" 1] ["the" 2] ["man" 1]}
(into #{} (exec-drpc drpc "all-tuples" "man"))))
- (feed feeder [["the foo"]])
+ (.feed feeder [["the foo"]])
(is (= #{["hello" 1] ["said" 1] ["the" 3] ["man" 1] ["foo" 1]}
(into #{} (exec-drpc drpc "all-tuples" "man")))))))))
(deftest test-word-count
- (t/with-local-cluster [cluster]
- (with-drpc [drpc]
+ (with-open [cluster (LocalCluster. )]
+ (with-open [drpc (LocalDRPC.)]
(letlocals
(bind topo (TridentTopology.))
- (bind feeder (feeder-spout ["sentence"]))
+ (bind feeder (FeederBatchSpout. ["sentence"]))
(bind word-counts
(-> topo
(.newStream "tester" feeder)
- (.each (fields "sentence") (Split.) (fields "word"))
- (.groupBy (fields "word"))
- (.persistentAggregate (memory-map-state) (Count.) (fields "count"))
+ (.each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
+ (.persistentAggregate (MemoryMapState$Factory.) (Count.) (Fields. ["count"]))
(.parallelismHint 6)
))
(-> topo
(.newDRPCStream "words" drpc)
- (.each (fields "args") (Split.) (fields "word"))
- (.groupBy (fields "word"))
- (.stateQuery word-counts (fields "word") (MapGet.) (fields "count"))
- (.aggregate (fields "count") (Sum.) (fields "sum"))
- (.project (fields "sum")))
- (with-topology [cluster topo storm-topo]
- (feed feeder [["hello the man said"] ["the"]])
+ (.each (Fields. ["args"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
+ (.stateQuery word-counts (Fields. ["word"]) (MapGet.) (Fields. ["count"]))
+ (.aggregate (Fields. ["count"]) (Sum.) (Fields. ["sum"]))
+ (.project (Fields. ["sum"])))
+ (with-open [storm-topo (.submitTopology cluster "testing" {} (.build topo))]
+ (.feed feeder [["hello the man said"] ["the"]])
(is (= [[2]] (exec-drpc drpc "words" "the")))
(is (= [[1]] (exec-drpc drpc "words" "hello")))
- (feed feeder [["the man on the moon"] ["where are you"]])
+ (.feed feeder [["the man on the moon"] ["where are you"]])
(is (= [[4]] (exec-drpc drpc "words" "the")))
(is (= [[2]] (exec-drpc drpc "words" "man")))
(is (= [[8]] (exec-drpc drpc "words" "man where you the")))
@@ -101,56 +106,56 @@
;; this test reproduces a bug where committer spouts freeze processing when
;; there's at least one repartitioning after the spout
(deftest test-word-count-committer-spout
- (t/with-local-cluster [cluster]
- (with-drpc [drpc]
+ (with-open [cluster (LocalCluster. )]
+ (with-open [drpc (LocalDRPC.)]
(letlocals
(bind topo (TridentTopology.))
- (bind feeder (feeder-committer-spout ["sentence"]))
+ (bind feeder (FeederCommitterBatchSpout. ["sentence"]))
(.setWaitToEmit feeder false) ;;this causes lots of empty batches
(bind word-counts
(-> topo
(.newStream "tester" feeder)
(.parallelismHint 2)
- (.each (fields "sentence") (Split.) (fields "word"))
- (.groupBy (fields "word"))
- (.persistentAggregate (memory-map-state) (Count.) (fields "count"))
+ (.each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
+ (.persistentAggregate (MemoryMapState$Factory.) (Count.) (Fields. ["count"]))
(.parallelismHint 6)
))
(-> topo
(.newDRPCStream "words" drpc)
- (.each (fields "args") (Split.) (fields "word"))
- (.groupBy (fields "word"))
- (.stateQuery word-counts (fields "word") (MapGet.) (fields "count"))
- (.aggregate (fields "count") (Sum.) (fields "sum"))
- (.project (fields "sum")))
- (with-topology [cluster topo storm-topo]
- (feed feeder [["hello the man said"] ["the"]])
+ (.each (Fields. ["args"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
+ (.stateQuery word-counts (Fields. ["word"]) (MapGet.) (Fields. ["count"]))
+ (.aggregate (Fields. ["count"]) (Sum.) (Fields. ["sum"]))
+ (.project (Fields. ["sum"])))
+ (with-open [storm-topo (.submitTopology cluster "testing" {} (.build topo))]
+ (.feed feeder [["hello the man said"] ["the"]])
(is (= [[2]] (exec-drpc drpc "words" "the")))
(is (= [[1]] (exec-drpc drpc "words" "hello")))
(Thread/sleep 1000) ;; this is necessary to reproduce the bug where committer spouts freeze processing
- (feed feeder [["the man on the moon"] ["where are you"]])
+ (.feed feeder [["the man on the moon"] ["where are you"]])
(is (= [[4]] (exec-drpc drpc "words" "the")))
(is (= [[2]] (exec-drpc drpc "words" "man")))
(is (= [[8]] (exec-drpc drpc "words" "man where you the")))
- (feed feeder [["the the"]])
+ (.feed feeder [["the the"]])
(is (= [[6]] (exec-drpc drpc "words" "the")))
- (feed feeder [["the"]])
+ (.feed feeder [["the"]])
(is (= [[7]] (exec-drpc drpc "words" "the")))
)))))
(deftest test-count-agg
- (t/with-local-cluster [cluster]
- (with-drpc [drpc]
+ (with-open [cluster (LocalCluster. )]
+ (with-open [drpc (LocalDRPC.)]
(letlocals
(bind topo (TridentTopology.))
(-> topo
(.newDRPCStream "numwords" drpc)
- (.each (fields "args") (Split.) (fields "word"))
- (.aggregate (CountAsAggregator.) (fields "count"))
+ (.each (Fields. ["args"]) (Split.) (Fields. ["word"]))
+ (.aggregate (CountAsAggregator.) (Fields. ["count"]))
(.parallelismHint 2) ;;this makes sure batchGlobal is working correctly
- (.project (fields "count")))
- (with-topology [cluster topo storm-topo]
+ (.project (Fields. ["count"])))
+ (with-open [storm-topo (.submitTopology cluster "testing" {} (.build topo))]
(doseq [i (range 100)]
(is (= [[1]] (exec-drpc drpc "numwords" "the"))))
(is (= [[0]] (exec-drpc drpc "numwords" "")))
@@ -158,76 +163,76 @@
)))))
(deftest test-split-merge
- (t/with-local-cluster [cluster]
- (with-drpc [drpc]
+ (with-open [cluster (LocalCluster. )]
+ (with-open [drpc (LocalDRPC.)]
(letlocals
(bind topo (TridentTopology.))
(bind drpc-stream (-> topo (.newDRPCStream "splitter" drpc)))
(bind s1
(-> drpc-stream
- (.each (fields "args") (Split.) (fields "word"))
- (.project (fields "word"))))
+ (.each (Fields. ["args"]) (Split.) (Fields. ["word"]))
+ (.project (Fields. ["word"]))))
(bind s2
(-> drpc-stream
- (.each (fields "args") (StringLength.) (fields "len"))
- (.project (fields "len"))))
+ (.each (Fields. ["args"]) (StringLength.) (Fields. ["len"]))
+ (.project (Fields. ["len"]))))
(.merge topo [s1 s2])
- (with-topology [cluster topo storm-topo]
- (is (t/ms= [[7] ["the"] ["man"]] (exec-drpc drpc "splitter" "the man")))
- (is (t/ms= [[5] ["hello"]] (exec-drpc drpc "splitter" "hello")))
+ (with-open [storm-topo (.submitTopology cluster "testing" {} (.build topo))]
+ (is (Testing/multiseteq [[7] ["the"] ["man"]] (exec-drpc drpc "splitter" "the man")))
+ (is (Testing/multiseteq [[5] ["hello"]] (exec-drpc drpc "splitter" "hello")))
)))))
(deftest test-multiple-groupings-same-stream
- (t/with-local-cluster [cluster]
- (with-drpc [drpc]
+ (with-open [cluster (LocalCluster. )]
+ (with-open [drpc (LocalDRPC.)]
(letlocals
(bind topo (TridentTopology.))
(bind drpc-stream (-> topo (.newDRPCStream "tester" drpc)
- (.each (fields "args") (TrueFilter.))))
+ (.each (Fields. ["args"]) (TrueFilter.))))
(bind s1
(-> drpc-stream
- (.groupBy (fields "args"))
- (.aggregate (CountAsAggregator.) (fields "count"))))
+ (.groupBy (Fields. ["args"]))
+ (.aggregate (CountAsAggregator.) (Fields. ["count"]))))
(bind s2
(-> drpc-stream
- (.groupBy (fields "args"))
- (.aggregate (CountAsAggregator.) (fields "count"))))
+ (.groupBy (Fields. ["args"]))
+ (.aggregate (CountAsAggregator.) (Fields. ["count"]))))
(.merge topo [s1 s2])
- (with-topology [cluster topo storm-topo]
- (is (t/ms= [["the" 1] ["the" 1]] (exec-drpc drpc "tester" "the")))
- (is (t/ms= [["aaaaa" 1] ["aaaaa" 1]] (exec-drpc drpc "tester" "aaaaa")))
+ (with-open [storm-topo (.submitTopology cluster "testing" {} (.build topo))]
+ (is (Testing/multiseteq [["the" 1] ["the" 1]] (exec-drpc drpc "tester" "the")))
+ (is (Testing/multiseteq [["aaaaa" 1] ["aaaaa" 1]] (exec-drpc drpc "tester" "aaaaa")))
)))))
(deftest test-multi-repartition
- (t/with-local-cluster [cluster]
- (with-drpc [drpc]
+ (with-open [cluster (LocalCluster. )]
+ (with-open [drpc (LocalDRPC.)]
(letlocals
(bind topo (TridentTopology.))
(bind drpc-stream (-> topo (.newDRPCStream "tester" drpc)
- (.each (fields "args") (Split.) (fields "word"))
+ (.each (Fields. ["args"]) (Split.) (Fields. ["word"]))
(.localOrShuffle)
(.shuffle)
- (.aggregate (CountAsAggregator.) (fields "count"))
+ (.aggregate (CountAsAggregator.) (Fields. ["count"]))
))
- (with-topology [cluster topo storm-topo]
- (is (t/ms= [[2]] (exec-drpc drpc "tester" "the man")))
- (is (t/ms= [[1]] (exec-drpc drpc "tester" "aaa")))
+ (with-open [storm-topo (.submitTopology cluster "testing" {} (.build topo))]
+ (is (Testing/multiseteq [[2]] (exec-drpc drpc "tester" "the man")))
+ (is (Testing/multiseteq [[1]] (exec-drpc drpc "tester" "aaa")))
)))))
(deftest test-stream-projection-validation
- (t/with-local-cluster [cluster]
+ (with-open [cluster (LocalCluster. )]
(letlocals
- (bind feeder (feeder-committer-spout ["sentence"]))
+ (bind feeder (FeederCommitterBatchSpout. ["sentence"]))
(bind topo (TridentTopology.))
;; valid projection fields will not throw exceptions
(bind word-counts
(-> topo
(.newStream "tester" feeder)
- (.each (fields "sentence") (Split.) (fields "word"))
- (.groupBy (fields "word"))
- (.persistentAggregate (memory-map-state) (Count.) (fields "count"))
+ (.each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
+ (.persistentAggregate (MemoryMapState$Factory.) (Count.) (Fields. ["count"]))
(.parallelismHint 6)
))
(bind stream (-> topo
@@ -235,63 +240,63 @@
;; test .each
(is (thrown? IllegalArgumentException
(-> stream
- (.each (fields "sentence1") (Split.) (fields "word")))))
+ (.each (Fields. ["sentence1"]) (Split.) (Fields. ["word"])))))
;; test .groupBy
(is (thrown? IllegalArgumentException
(-> stream
- (.each (fields "sentence") (Split.) (fields "word"))
- (.groupBy (fields "word1")))))
+ (.each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word1"])))))
;; test .aggregate
(is (thrown? IllegalArgumentException
(-> stream
- (.each (fields "sentence") (Split.) (fields "word"))
- (.groupBy (fields "word"))
- (.aggregate (fields "word1") (Count.) (fields "count")))))
+ (.each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
+ (.aggregate (Fields. ["word1"]) (Count.) (Fields. ["count"])))))
;; test .project
(is (thrown? IllegalArgumentException
(-> stream
- (.project (fields "sentence1")))))
+ (.project (Fields. ["sentence1"])))))
;; test .partitionBy
(is (thrown? IllegalArgumentException
(-> stream
- (.partitionBy (fields "sentence1")))))
+ (.partitionBy (Fields. ["sentence1"])))))
;; test .partitionAggregate
(is (thrown? IllegalArgumentException
(-> stream
- (.each (fields "sentence") (Split.) (fields "word"))
- (.partitionAggregate (fields "word1") (Count.) (fields "count")))))
+ (.each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
+ (.partitionAggregate (Fields. ["word1"]) (Count.) (Fields. ["count"])))))
;; test .persistentAggregate
(is (thrown? IllegalArgumentException
(-> stream
- (.each (fields "sentence") (Split.) (fields "word"))
- (.groupBy (fields "word"))
- (.persistentAggregate (StateSpec. (MemoryMapState$Factory.)) (fields "non-existent") (Count.) (fields "count")))))
+ (.each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
+ (.persistentAggregate (StateSpec. (MemoryMapState$Factory.)) (Fields. ["non-existent"]) (Count.) (Fields. ["count"])))))
;; test .partitionPersist
(is (thrown? IllegalArgumentException
(-> stream
- (.each (fields "sentence") (Split.) (fields "word"))
- (.groupBy (fields "word"))
+ (.each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
(.partitionPersist (StateSpec. (MemoryMapState$Factory.))
- (fields "non-existent")
+ (Fields. ["non-existent"])
(CombinerAggStateUpdater. (Count.))
- (fields "count")))))
+ (Fields. ["count"])))))
;; test .stateQuery
- (with-drpc [drpc]
+ (with-open [drpc (LocalDRPC.)]
(is (thrown? IllegalArgumentException
(-> topo
(.newDRPCStream "words" drpc)
- (.each (fields "args") (Split.) (fields "word"))
- (.groupBy (fields "word"))
- (.stateQuery word-counts (fields "word1") (MapGet.) (fields "count"))))))
+ (.each (Fields. ["args"]) (Split.) (Fields. ["word"]))
+ (.groupBy (Fields. ["word"]))
+ (.stateQuery word-counts (Fields. ["word1"]) (MapGet.) (Fields. ["count"]))))))
)))
(deftest test-set-component-resources
- (t/with-local-cluster [cluster]
- (with-drpc [drpc]
+ (with-open [cluster (LocalCluster. )]
+ (with-open [drpc (LocalDRPC.)]
(letlocals
(bind topo (TridentTopology.))
- (bind feeder (feeder-spout ["sentence"]))
+ (bind feeder (FeederBatchSpout. ["sentence"]))
(bind add-bang (proxy [BaseFunction] []
(execute [tuple collector]
(. collector emit (str (. tuple getString 0) "!")))))
@@ -304,18 +309,18 @@
(parallelismHint 5)
(setCPULoad 20)
(setMemoryLoad 512 256)
- (each (fields "sentence") (Split.) (fields "word"))
+ (each (Fields. ["sentence"]) (Split.) (Fields. ["word"]))
(setCPULoad 10)
(setMemoryLoad 512)
- (each (fields "word") add-bang (fields "word!"))
+ (each (Fields. ["word"]) add-bang (Fields. ["word!"]))
(parallelismHint 10)
(setCPULoad 50)
(setMemoryLoad 1024)
- (groupBy (fields "word!"))
- (persistentAggregate (memory-map-state) (Count.) (fields "count"))
+ (groupBy (Fields. ["word!"]))
+ (persistentAggregate (MemoryMapState$Factory.) (Count.) (Fields. ["count"]))
(setCPULoad 100)
(setMemoryLoad 2048)))
- (with-topology [cluster topo storm-topo]
+ (with-open [storm-topo (.submitTopology cluster "testing" {} (.build topo))]
(let [parse-fn (fn [[k v]]
[k (clojurify-structure (. (JSONParser.) parse (.. v get_common get_json_conf)))])
@@ -364,23 +369,4 @@
(get TOPOLOGY-COMPONENT-CPU-PCORE-PERCENT))
100.0)))))))))
-;; (deftest test-split-merge
-;; (t/with-local-cluster [cluster]
-;; (with-drpc [drpc]
-;; (letlocals
-;; (bind topo (TridentTopology.))
-;; (bind drpc-stream (-> topo (.newDRPCStream "splitter" drpc)))
-;; (bind s1
-;; (-> drpc-stream
-;; (.each (fields "args") (Split.) (fields "word"))
-;; (.project (fields "word"))))
-;; (bind s2
-;; (-> drpc-stream
-;; (.each (fields "args") (StringLength.) (fields "len"))
-;; (.project (fields "len"))))
-;;
-;; (.merge topo [s1 s2])
-;; (with-topology [cluster topo]
-;; (is (t/ms= [[7] ["the"] ["man"]] (exec-drpc drpc "splitter" "the man")))
-;; (is (t/ms= [[5] ["hello"]] (exec-drpc drpc "splitter" "hello")))
-;; )))))
+
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/clojure_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/clojure_test.clj b/storm-core/test/clj/org/apache/storm/clojure_test.clj
deleted file mode 100644
index 13fdeb7..0000000
--- a/storm-core/test/clj/org/apache/storm/clojure_test.clj
+++ /dev/null
@@ -1,159 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.clojure-test
- (:use [clojure test])
- (:import [org.apache.storm.testing TestWordSpout TestPlannerSpout]
- [org.apache.storm.tuple Fields])
- (:use [org.apache.storm testing config])
- (:use [org.apache.storm.internal clojure])
- (:use [org.apache.storm.daemon common])
- (:require [org.apache.storm.internal [thrift :as thrift]])
- (:import [org.apache.storm Thrift])
- (:import [org.apache.storm.utils Utils]))
-
-(defbolt lalala-bolt1 ["word"] [[val :as tuple] collector]
- (let [ret (str val "lalala")]
- (emit-bolt! collector [ret] :anchor tuple)
- (ack! collector tuple)
- ))
-
-(defbolt lalala-bolt2 ["word"] {:prepare true}
- [conf context collector]
- (let [state (atom nil)]
- (reset! state "lalala")
- (bolt
- (execute [tuple]
- (let [ret (-> (.getValue tuple 0) (str @state))]
- (emit-bolt! collector [ret] :anchor tuple)
- (ack! collector tuple)
- ))
- )))
-
-(defbolt lalala-bolt3 ["word"] {:prepare true :params [prefix]}
- [conf context collector]
- (let [state (atom nil)]
- (bolt
- (prepare [_ _ _]
- (reset! state (str prefix "lalala")))
- (execute [{val "word" :as tuple}]
- (let [ret (-> (.getValue tuple 0) (str @state))]
- (emit-bolt! collector [ret] :anchor tuple)
- (ack! collector tuple)
- )))
- ))
-
-(deftest test-clojure-bolt
- (with-simulated-time-local-cluster [cluster :supervisors 4]
- (let [nimbus (:nimbus cluster)
- topology (Thrift/buildTopology
- {"1" (Thrift/prepareSpoutDetails (TestWordSpout. false))}
- {"2" (Thrift/prepareBoltDetails
- {(Utils/getGlobalStreamId "1" nil)
- (Thrift/prepareShuffleGrouping)}
- lalala-bolt1)
- "3" (Thrift/prepareBoltDetails
- {(Utils/getGlobalStreamId "1" nil)
- (Thrift/prepareLocalOrShuffleGrouping)}
- lalala-bolt2)
- "4" (Thrift/prepareBoltDetails
- {(Utils/getGlobalStreamId "1" nil)
- (Thrift/prepareShuffleGrouping)}
- (lalala-bolt3 "_nathan_"))}
- )
- results (complete-topology cluster
- topology
- :mock-sources {"1" [["david"]
- ["adam"]
- ]}
- )]
- (is (ms= [["davidlalala"] ["adamlalala"]] (read-tuples results "2")))
- (is (ms= [["davidlalala"] ["adamlalala"]] (read-tuples results "3")))
- (is (ms= [["david_nathan_lalala"] ["adam_nathan_lalala"]] (read-tuples results "4")))
- )))
-
-(defbolt punctuator-bolt ["word" "period" "question" "exclamation"]
- [tuple collector]
- (if (= (:word tuple) "bar")
- (do
- (emit-bolt! collector {:word "bar" :period "bar" :question "bar"
- "exclamation" "bar"})
- (ack! collector tuple))
- (let [ res (assoc tuple :period (str (:word tuple) "."))
- res (assoc res :exclamation (str (:word tuple) "!"))
- res (assoc res :question (str (:word tuple) "?")) ]
- (emit-bolt! collector res)
- (ack! collector tuple))))
-
-(deftest test-map-emit
- (with-simulated-time-local-cluster [cluster :supervisors 4]
- (let [topology (Thrift/buildTopology
- {"words" (Thrift/prepareSpoutDetails (TestWordSpout. false))}
- {"out" (Thrift/prepareBoltDetails
- {(Utils/getGlobalStreamId "words" nil)
- (Thrift/prepareShuffleGrouping)}
- punctuator-bolt)})
- results (complete-topology cluster
- topology
- :mock-sources {"words" [["foo"] ["bar"]]}
- )]
- (is (ms= [["foo" "foo." "foo?" "foo!"]
- ["bar" "bar" "bar" "bar"]] (read-tuples results "out"))))))
-
-(defbolt conf-query-bolt ["conf" "val"] {:prepare true :params [conf] :conf conf}
- [conf context collector]
- (bolt
- (execute [tuple]
- (let [name (.getValue tuple 0)
- val (if (= name "!MAX_MSG_TIMEOUT") (.maxTopologyMessageTimeout context) (get conf name))]
- (emit-bolt! collector [name val] :anchor tuple)
- (ack! collector tuple))
- )))
-
-(deftest test-component-specific-config-clojure
- (with-simulated-time-local-cluster [cluster]
- (let [topology (Thrift/buildTopology
- {"1" (Thrift/prepareSpoutDetails
- (TestPlannerSpout. (Fields. ["conf"]))
- nil
- {TOPOLOGY-MESSAGE-TIMEOUT-SECS 40})}
- {"2" (Thrift/prepareBoltDetails
- {(Utils/getGlobalStreamId "1" nil)
- (Thrift/prepareShuffleGrouping)}
- (conf-query-bolt {"fake.config" 1
- TOPOLOGY-MAX-TASK-PARALLELISM 2
- TOPOLOGY-MAX-SPOUT-PENDING 10})
- nil
- {TOPOLOGY-MAX-SPOUT-PENDING 3})})
- results (complete-topology cluster
- topology
- :topology-name "test123"
- :storm-conf {TOPOLOGY-MAX-TASK-PARALLELISM 10
- TOPOLOGY-MESSAGE-TIMEOUT-SECS 30}
- :mock-sources {"1" [["fake.config"]
- [TOPOLOGY-MAX-TASK-PARALLELISM]
- [TOPOLOGY-MAX-SPOUT-PENDING]
- ["!MAX_MSG_TIMEOUT"]
- [TOPOLOGY-NAME]
- ]})]
- (is (= {"fake.config" 1
- TOPOLOGY-MAX-TASK-PARALLELISM 2
- TOPOLOGY-MAX-SPOUT-PENDING 3
- "!MAX_MSG_TIMEOUT" 40
- TOPOLOGY-NAME "test123"}
- (->> (read-tuples results "2")
- (apply concat)
- (apply hash-map))
- )))))
[10/10] storm git commit: Added STORM-1281 to Changelog
Posted by bo...@apache.org.
Added STORM-1281 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/607ba8a9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/607ba8a9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/607ba8a9
Branch: refs/heads/master
Commit: 607ba8a9e5c9ca939b4317cbf839cbc52d991799
Parents: 557c49d
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Sat Dec 3 09:49:28 2016 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Sat Dec 3 09:49:28 2016 -0600
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/607ba8a9/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fd8bd13..2b88cc0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-1281: LocalCluster, testing4j and testing.clj to java
* STORM-2226: Fix kafka spout offset lag ui for kerberized kafka
* STORM-1276: line for line translation of nimbus to java
* STORM-2224: Exposed a method to override in computing the field from given tuple in FieldSelector
[07/10] storm git commit: STORM-1281: LocalCluster,
testing4j and testing.clj to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
index 431b9d8..45014fc 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.ShellSpout;
import org.apache.storm.task.ShellBolt;
@@ -110,12 +111,10 @@ public class WordCountTopologyNode {
else {
conf.setMaxTaskParallelism(3);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("word-count", conf, builder.createTopology());
-
- Thread.sleep(10000);
-
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("word-count", conf, builder.createTopology());) {
+ Thread.sleep(10000);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java
index ba18f7c..70a23b8 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java
@@ -20,6 +20,7 @@ package org.apache.storm.starter.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.hbase.trident.windowing.HBaseWindowsStoreFactory;
@@ -77,12 +78,10 @@ public class TridentHBaseWindowingStoreTopology {
HBaseWindowsStoreFactory windowStoreFactory = new HBaseWindowsStoreFactory(new HashMap<String, Object>(), "window-state", "cf".getBytes("UTF-8"), "tuples".getBytes("UTF-8"));
if (args.length == 0) {
- LocalCluster cluster = new LocalCluster();
- String topologyName = "wordCounterWithWindowing";
- cluster.submitTopology(topologyName, conf, buildTopology(windowStoreFactory));
- Utils.sleep(120 * 1000);
- cluster.killTopology(topologyName);
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("wordCounterWithWindowing", conf, buildTopology(windowStoreFactory));) {
+ Utils.sleep(120 * 1000);
+ }
System.exit(0);
} else {
conf.setNumWorkers(3);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
index c3661a2..5ddace8 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.LocalDRPC;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -108,14 +109,14 @@ public class TridentMapExample {
Config conf = new Config();
conf.setMaxSpoutPending(20);
if (args.length == 0) {
- LocalDRPC drpc = new LocalDRPC();
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
- for (int i = 0; i < 100; i++) {
- System.out.println("DRPC RESULT: " + drpc.execute("words", "CAT THE DOG JUMPED"));
- Thread.sleep(1000);
+ try (LocalDRPC drpc = new LocalDRPC();
+ LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(drpc));) {
+ for (int i = 0; i < 100; i++) {
+ System.out.println("DRPC RESULT: " + drpc.execute("words", "CAT THE DOG JUMPED"));
+ Thread.sleep(1000);
+ }
}
- cluster.shutdown();
} else {
conf.setNumWorkers(3);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
index d985436..e5a775b 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.starter.spout.RandomNumberGeneratorSpout;
@@ -113,10 +114,10 @@ public class TridentMinMaxOfDevicesTopology {
Config conf = new Config();
conf.setMaxSpoutPending(20);
if (args.length == 0) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("devices-topology", conf, topology);
- Utils.sleep(60 * 1000);
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("devices-topology", conf, topology);) {
+ Utils.sleep(60 * 1000);
+ }
System.exit(0);
} else {
conf.setNumWorkers(3);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
index 192b198..947b64b 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.trident.Stream;
@@ -94,10 +95,10 @@ public class TridentMinMaxOfVehiclesTopology {
Config conf = new Config();
conf.setMaxSpoutPending(20);
if (args.length == 0) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("vehicles-topology", conf, topology);
- Utils.sleep(60 * 1000);
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("vehicles-topology", conf, topology);) {
+ Utils.sleep(60 * 1000);
+ }
System.exit(0);
} else {
conf.setNumWorkers(3);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
index 056b2b6..6533a4e 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.LocalDRPC;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.task.IMetricsContext;
@@ -136,21 +137,17 @@ public class TridentReach {
}
public static void main(String[] args) throws Exception {
- LocalDRPC drpc = new LocalDRPC();
Config conf = new Config();
- LocalCluster cluster = new LocalCluster();
+ try (LocalDRPC drpc = new LocalDRPC();
+ LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("reach", conf, buildTopology(drpc));) {
- cluster.submitTopology("reach", conf, buildTopology(drpc));
+ Thread.sleep(2000);
- Thread.sleep(2000);
-
- System.out.println("REACH: " + drpc.execute("reach", "aaa"));
- System.out.println("REACH: " + drpc.execute("reach", "foo.com/blog/1"));
- System.out.println("REACH: " + drpc.execute("reach", "engineering.twitter.com/blog/5"));
-
-
- cluster.shutdown();
- drpc.shutdown();
+ System.out.println("REACH: " + drpc.execute("reach", "aaa"));
+ System.out.println("REACH: " + drpc.execute("reach", "foo.com/blog/1"));
+ System.out.println("REACH: " + drpc.execute("reach", "engineering.twitter.com/blog/5"));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
index 5aec01d..c43b4b0 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
@@ -20,6 +20,7 @@ package org.apache.storm.starter.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.topology.base.BaseWindowedBolt;
@@ -83,10 +84,10 @@ public class TridentWindowingInmemoryStoreTopology {
);
for (WindowConfig windowConfig : list) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("wordCounter", conf, buildTopology(mapState, windowConfig));
- Utils.sleep(60 * 1000);
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(mapState, windowConfig));) {
+ Utils.sleep(60 * 1000);
+ }
}
System.exit(0);
} else {
@@ -94,5 +95,4 @@ public class TridentWindowingInmemoryStoreTopology {
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(mapState, SlidingCountWindow.of(1000, 100)));
}
}
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
index ac46474..553c26f 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.LocalDRPC;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -70,12 +71,13 @@ public class TridentWordCount {
Config conf = new Config();
conf.setMaxSpoutPending(20);
if (args.length == 0) {
- LocalDRPC drpc = new LocalDRPC();
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
- for (int i = 0; i < 100; i++) {
- System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
- Thread.sleep(1000);
+ try (LocalDRPC drpc = new LocalDRPC();
+ LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(drpc));) {
+ for (int i = 0; i < 100; i++) {
+ System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
+ Thread.sleep(1000);
+ }
}
}
else {
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/util/StormRunner.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/util/StormRunner.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/util/StormRunner.java
index d7f2bf4..06ec4d7 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/util/StormRunner.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/util/StormRunner.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter.util;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
@@ -34,11 +35,12 @@ public final class StormRunner {
public static void runTopologyLocally(StormTopology topology, String topologyName, Config conf, int runtimeInSeconds)
throws InterruptedException {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology(topologyName, conf, topology);
- Thread.sleep((long) runtimeInSeconds * MILLIS_IN_SEC);
- cluster.killTopology(topologyName);
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology(topologyName, conf, topology);) {
+ Thread.sleep((long) runtimeInSeconds * MILLIS_IN_SEC);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
public static void runTopologyRemotely(StormTopology topology, String topologyName, Config conf)
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/storm/starter/StatefulTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/storm/starter/StatefulTopology.java b/examples/storm-starter/src/jvm/storm/starter/StatefulTopology.java
index ba513dd..d073350 100644
--- a/examples/storm-starter/src/jvm/storm/starter/StatefulTopology.java
+++ b/examples/storm-starter/src/jvm/storm/starter/StatefulTopology.java
@@ -19,6 +19,7 @@ package storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.starter.spout.RandomIntegerSpout;
@@ -134,12 +135,10 @@ public class StatefulTopology {
conf.setNumWorkers(1);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
} else {
- LocalCluster cluster = new LocalCluster();
- StormTopology topology = builder.createTopology();
- cluster.submitTopology("test", conf, topology);
- Utils.sleep(40000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topology = cluster.submitTopology("test", conf, builder.createTopology());) {
+ Utils.sleep(40000);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/storm/starter/StatefulWindowingTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/storm/starter/StatefulWindowingTopology.java b/examples/storm-starter/src/jvm/storm/starter/StatefulWindowingTopology.java
index e18ef36..e43759c 100644
--- a/examples/storm-starter/src/jvm/storm/starter/StatefulWindowingTopology.java
+++ b/examples/storm-starter/src/jvm/storm/starter/StatefulWindowingTopology.java
@@ -19,6 +19,7 @@ package storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.starter.bolt.PrinterBolt;
@@ -101,12 +102,10 @@ public class StatefulWindowingTopology {
conf.setNumWorkers(1);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
} else {
- LocalCluster cluster = new LocalCluster();
- StormTopology topology = builder.createTopology();
- cluster.submitTopology("test", conf, topology);
- Utils.sleep(40000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology());) {
+ Utils.sleep(40000);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/external/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java
----------------------------------------------------------------------
diff --git a/external/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java b/external/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java
index 5848d2f..982765f 100644
--- a/external/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java
+++ b/external/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java
@@ -19,6 +19,7 @@ package org.apache.storm.flux;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.generated.SubmitOptions;
@@ -204,11 +205,11 @@ public class Flux {
} else {
cluster = new LocalCluster();
}
- cluster.submitTopology(topologyName, conf, topology);
-
- Utils.sleep(sleepTime);
- cluster.killTopology(topologyName);
- cluster.shutdown();
+ try (LocalTopology topo = cluster.submitTopology(topologyName, conf, topology)) {
+ Utils.sleep(sleepTime);
+ } finally {
+ cluster.shutdown();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java b/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
index c1ceeba..5755de8 100644
--- a/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
+++ b/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
@@ -29,9 +29,8 @@ import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
import org.apache.calcite.rel.type.RelDataTypeSystem;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.storm.Config;
-import org.apache.storm.ILocalCluster;
import org.apache.storm.LocalCluster;
-import org.apache.storm.generated.StormTopology;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.sql.TestUtils;
import org.apache.storm.sql.compiler.TestCompilerUtils;
import org.apache.storm.sql.runtime.ISqlTridentDataSource;
@@ -40,8 +39,10 @@ import org.apache.storm.trident.TridentTopology;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import java.time.ZoneOffset;
@@ -57,6 +58,20 @@ public class TestPlanCompiler {
private final JavaTypeFactory typeFactory = new JavaTypeFactoryImpl(
RelDataTypeSystem.DEFAULT);
private final DataContext dataContext = new StormDataContext();
+ private static LocalCluster cluster;
+
+ @BeforeClass
+ public static void staticSetup() throws Exception {
+ cluster = new LocalCluster();
+ }
+
+ @AfterClass
+ public static void staticCleanup() {
+ if (cluster!= null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
@Before
public void setUp() {
@@ -347,11 +362,8 @@ public class TestPlanCompiler {
final Config conf = new Config();
conf.setMaxSpoutPending(20);
- ILocalCluster cluster = new LocalCluster();
- StormTopology stormTopo = topo.build();
- try {
- Utils.setClassLoaderForJavaDeSerialize(proc.getClass().getClassLoader());
- cluster.submitTopology("storm-sql", conf, stormTopo);
+ Utils.setClassLoaderForJavaDeSerialize(proc.getClass().getClassLoader());
+ try (LocalTopology stormTopo = cluster.submitTopology("storm-sql", conf, topo.build())) {
waitForCompletion(1000 * 1000, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
@@ -359,8 +371,10 @@ public class TestPlanCompiler {
}
});
} finally {
+ while(cluster.getClusterInfo().get_topologies_size() > 0) {
+ Thread.sleep(10);
+ }
Utils.resetClassLoaderForJavaDeSerialize();
- cluster.shutdown();
}
}
@@ -375,4 +389,4 @@ public class TestPlanCompiler {
Assert.assertTrue("Two lists are not same (even ignoring order)!\n"+ "Expected: " + expected + "\n" + "Actual: " + actual,
CollectionUtils.isEqualCollection(expected, actual));
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTopology.java
----------------------------------------------------------------------
diff --git a/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTopology.java b/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTopology.java
index 88b2bf1..298353c 100644
--- a/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTopology.java
+++ b/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTopology.java
@@ -20,6 +20,7 @@ package org.apache.storm.druid;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.druid.bolt.DruidBeamBolt;
import org.apache.storm.druid.bolt.DruidBeamFactory;
@@ -69,12 +70,10 @@ public class SampleDruidBoltTopology {
} else {
conf.setMaxTaskParallelism(3);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("druid-test", conf, topologyBuilder.createTopology());
-
- Thread.sleep(30000);
-
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("druid-test", conf, topologyBuilder.createTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTridentTopology.java
----------------------------------------------------------------------
diff --git a/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTridentTopology.java b/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTridentTopology.java
index 0e20ecd..3de18c1 100644
--- a/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTridentTopology.java
+++ b/external/storm-druid/src/test/java/org/apache/storm/druid/SampleDruidBoltTridentTopology.java
@@ -20,6 +20,7 @@ package org.apache.storm.druid;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.druid.bolt.DruidBeamFactory;
import org.apache.storm.druid.bolt.ITupleDruidEventMapper;
@@ -78,12 +79,10 @@ public class SampleDruidBoltTridentTopology {
} else {
conf.setMaxTaskParallelism(3);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("druid-test", conf, tridentTopology.build());
-
- Thread.sleep(30000);
-
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("druid-test", conf, tridentTopology.build());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/samples/EventCount.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/samples/EventCount.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/samples/EventCount.java
index 2c2261c..ae15634 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/samples/EventCount.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/samples/EventCount.java
@@ -20,6 +20,7 @@ package org.apache.storm.eventhubs.samples;
import org.apache.storm.StormSubmitter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.topology.TopologyBuilder;
@@ -135,14 +136,12 @@ public class EventCount {
} else {
config.setMaxTaskParallelism(2);
- LocalCluster localCluster = new LocalCluster();
- localCluster.submitTopology("test", config, topology);
-
- Thread.sleep(5000000);
-
- localCluster.shutdown();
+ try (LocalCluster localCluster = new LocalCluster();
+ LocalTopology topo = localCluster.submitTopology("test", config, topology);) {
+ Thread.sleep(5000000);
+ }
}
- }
+ }
protected void runScenario(String[] args) throws Exception{
readEHConfig(args);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/external/storm-jms/examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java b/external/storm-jms/examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
index 3324aac..82dbd5b 100644
--- a/external/storm-jms/examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
+++ b/external/storm-jms/examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
@@ -24,6 +24,7 @@ import javax.jms.TextMessage;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.jms.JmsMessageProducer;
import org.apache.storm.jms.JmsProvider;
@@ -120,11 +121,10 @@ public class ExampleJmsTopology {
conf.setDebug(true);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("storm-jms-example", conf, builder.createTopology());
- Utils.sleep(60000);
- cluster.killTopology("storm-jms-example");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("storm-jms-example", conf, builder.createTopology());) {
+ Utils.sleep(60000);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/test/KafkaSpoutTopologyMainNamedTopics.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/test/KafkaSpoutTopologyMainNamedTopics.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/test/KafkaSpoutTopologyMainNamedTopics.java
index 5a78137..10ad2c7 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/test/KafkaSpoutTopologyMainNamedTopics.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/test/KafkaSpoutTopologyMainNamedTopics.java
@@ -58,10 +58,9 @@ public class KafkaSpoutTopologyMainNamedTopics {
} else {
submitTopologyRemoteCluster(args[0], getTopolgyKafkaSpout(), getConfig());
}
-
}
- protected void submitTopologyLocalCluster(StormTopology topology, Config config) throws InterruptedException {
+ protected void submitTopologyLocalCluster(StormTopology topology, Config config) throws Exception {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("test", config, topology);
stopWaitingForInput();
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
index 0873601..049fce7 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
@@ -20,6 +20,7 @@ package org.apache.storm.kafka;
import org.junit.Test;
import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
import org.junit.After;
import org.junit.Before;
import static org.junit.Assert.assertEquals;
@@ -34,15 +35,19 @@ public class ExponentialBackoffMsgRetryManagerTest {
private static final Long TEST_OFFSET2 = 102L;
private static final Long TEST_OFFSET3 = 105L;
private static final Long TEST_NEW_OFFSET = 103L;
+ private SimulatedTime st;
@Before
public void setup() throws Exception {
- Time.startSimulating();
+ st = new SimulatedTime();
}
@After
public void cleanup() throws Exception {
- Time.stopSimulating();
+ if (st != null) {
+ st.close();
+ st = null;
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/external/storm-mqtt/core/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/core/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java b/external/storm-mqtt/core/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java
index 57725b4..0dd4d73 100644
--- a/external/storm-mqtt/core/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java
+++ b/external/storm-mqtt/core/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java
@@ -19,6 +19,7 @@ package org.apache.storm.mqtt;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.testing.IntegrationTest;
import org.apache.storm.topology.TopologyBuilder;
@@ -100,31 +101,31 @@ public class StormMqttIntegrationTest implements Serializable{
Topic[] topics = {new Topic("/integration-result", QoS.AT_LEAST_ONCE)};
byte[] qoses = connection.subscribe(topics);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", new Config(), buildMqttTopology());
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", new Config(), buildMqttTopology());) {
- LOG.info("topology started");
- while(!spoutActivated) {
- Thread.sleep(500);
- }
+ LOG.info("topology started");
+ while(!spoutActivated) {
+ Thread.sleep(500);
+ }
- // publish a retained message to the broker
- MqttOptions options = new MqttOptions();
- options.setCleanConnection(false);
- MqttPublisher publisher = new MqttPublisher(options, true);
- publisher.connectMqtt("MqttPublisher");
- publisher.publish(new MqttMessage(TEST_TOPIC, "test".getBytes()));
+ // publish a retained message to the broker
+ MqttOptions options = new MqttOptions();
+ options.setCleanConnection(false);
+ MqttPublisher publisher = new MqttPublisher(options, true);
+ publisher.connectMqtt("MqttPublisher");
+ publisher.publish(new MqttMessage(TEST_TOPIC, "test".getBytes()));
- LOG.info("published message");
+ LOG.info("published message");
- Message message = connection.receive();
- LOG.info("Message recieved on topic: {}", message.getTopic());
- LOG.info("Payload: {}", new String(message.getPayload()));
- message.ack();
+ Message message = connection.receive();
+ LOG.info("Message recieved on topic: {}", message.getTopic());
+ LOG.info("Payload: {}", new String(message.getPayload()));
+ message.ack();
- Assert.assertArrayEquals(message.getPayload(), RESULT_PAYLOAD.getBytes());
- Assert.assertEquals(message.getTopic(), RESULT_TOPIC);
- cluster.shutdown();
+ Assert.assertArrayEquals(message.getPayload(), RESULT_PAYLOAD.getBytes());
+ Assert.assertEquals(message.getTopic(), RESULT_TOPIC);
+ }
}
public StormTopology buildMqttTopology(){
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/integration-test/src/main/java/org/apache/storm/ExclamationTopology.java
----------------------------------------------------------------------
diff --git a/integration-test/src/main/java/org/apache/storm/ExclamationTopology.java b/integration-test/src/main/java/org/apache/storm/ExclamationTopology.java
index d464608..992bb5d 100644
--- a/integration-test/src/main/java/org/apache/storm/ExclamationTopology.java
+++ b/integration-test/src/main/java/org/apache/storm/ExclamationTopology.java
@@ -18,6 +18,7 @@
package org.apache.storm;
import org.apache.storm.generated.StormTopology;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.testing.TestWordSpout;
@@ -75,11 +76,10 @@ public class ExclamationTopology {
}
else {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", conf, topology);
- Utils.sleep(10000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopolgoy topo = cluster.submitTopology("test", conf, topology);) {
+ Utils.sleep(10000);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-clojure/src/clj/org/apache/storm/clojure.clj
----------------------------------------------------------------------
diff --git a/storm-clojure/src/clj/org/apache/storm/clojure.clj b/storm-clojure/src/clj/org/apache/storm/clojure.clj
index 607fc24..1fe0c26 100644
--- a/storm-clojure/src/clj/org/apache/storm/clojure.clj
+++ b/storm-clojure/src/clj/org/apache/storm/clojure.clj
@@ -195,6 +195,24 @@
(let [values (tuple-values values collector stream)]
(.emitDirect ^SpoutOutputCollector (:output-collector collector) task stream values id)))
+(defmacro defalias
+ "Defines an alias for a var: a new var with the same root binding (if
+ any) and similar metadata. The metadata of the alias is its initial
+ metadata (as provided by def) merged into the metadata of the original."
+ ([name orig]
+ `(do
+ (alter-meta!
+ (if (.hasRoot (var ~orig))
+ (def ~name (.getRawRoot (var ~orig)))
+ (def ~name))
+ ;; When copying metadata, disregard {:macro false}.
+ ;; Workaround for http://www.assembla.com/spaces/clojure/tickets/273
+ #(conj (dissoc % :macro)
+ (apply dissoc (meta (var ~orig)) (remove #{:macro} (keys %)))))
+ (var ~name)))
+ ([name orig doc]
+ (list `defalias (with-meta name (assoc (meta name) :doc doc)) orig)))
+
(defalias topology thrift/mk-topology)
(defalias bolt-spec thrift/mk-bolt-spec)
(defalias spout-spec thrift/mk-spout-spec)
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-clojure/src/clj/org/apache/storm/testing.clj
----------------------------------------------------------------------
diff --git a/storm-clojure/src/clj/org/apache/storm/testing.clj b/storm-clojure/src/clj/org/apache/storm/testing.clj
new file mode 100644
index 0000000..e080dd2
--- /dev/null
+++ b/storm-clojure/src/clj/org/apache/storm/testing.clj
@@ -0,0 +1,270 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+
+(ns org.apache.storm.testing
+ (:import [org.apache.storm LocalCluster$Builder])
+ (:import [java.util.function UnaryOperator])
+ (:import [org.apache.storm.utils Time Time$SimulatedTime Utils RegisteredGlobalState])
+ (:import [org.apache.storm.testing InProcessZookeeper MkTupleParam TestJob MkClusterParam
+ TrackedTopology CompleteTopologyParam MockedSources])
+ (:import [org.apache.storm Thrift Testing Testing$Condition])
+ (:import [org.apache.storm.testing MockLeaderElector])
+ (:import [org.json.simple JSONValue])
+ (:use [org.apache.storm util config log])
+ (:use [org.apache.storm thrift]))
+
+(defnk add-supervisor
+ [cluster-map :ports 2 :conf {} :id nil]
+ (let [local-cluster (:local-cluster cluster-map)]
+ (.addSupervisor local-cluster ports conf id)))
+
+(defnk mock-leader-elector [:is-leader true :leader-name "test-host" :leader-port 9999]
+ (MockLeaderElector. is-leader leader-name leader-port))
+
+(defn local-cluster-state [local-cluster]
+ {:nimbus (.getNimbus local-cluster)
+ :daemon-conf (.getDaemonConf local-cluster)
+ :storm-cluster-state (.getClusterState local-cluster)
+ :local-cluster local-cluster})
+
+(defnk mk-mocked-nimbus
+ [:daemon-conf {} :inimbus nil :blob-store nil :cluster-state nil
+ :leader-elector nil :group-mapper nil :nimbus-daemon false :nimbus-wrapper nil]
+ (let [builder (doto (LocalCluster$Builder.)
+ (.withDaemonConf daemon-conf)
+ (.withINimbus inimbus)
+ (.withBlobStore blob-store)
+ (.withClusterState cluster-state)
+ (.withLeaderElector leader-elector)
+ (.withGroupMapper group-mapper)
+ (.withNimbusDaemon nimbus-daemon)
+ (.withNimbusWrapper (when nimbus-wrapper (reify UnaryOperator (apply [this nimbus] (nimbus-wrapper nimbus))))))
+ local-cluster (.build builder)]
+ (local-cluster-state local-cluster)))
+
+(defnk mk-local-storm-cluster [:supervisors 2 :ports-per-supervisor 3 :daemon-conf {} :inimbus nil :group-mapper nil :supervisor-slot-port-min 1024 :nimbus-daemon false]
+ (let [builder (doto (LocalCluster$Builder.)
+ (.withSupervisors supervisors)
+ (.withPortsPerSupervisor ports-per-supervisor)
+ (.withDaemonConf daemon-conf)
+ (.withINimbus inimbus)
+ (.withGroupMapper group-mapper)
+ (.withSupervisorSlotPortMin supervisor-slot-port-min)
+ (.withNimbusDaemon nimbus-daemon))
+ local-cluster (.build builder)]
+ (local-cluster-state local-cluster)))
+
+(defn get-supervisor [cluster-map supervisor-id]
+ (let [local-cluster (:local-cluster cluster-map)]
+ (.getSupervisor local-cluster supervisor-id)))
+
+(defn kill-supervisor [cluster-map supervisor-id]
+ (let [local-cluster (:local-cluster cluster-map)]
+ (.killSupervisor local-cluster supervisor-id)))
+
+(defn kill-local-storm-cluster [cluster-map]
+ (let [local-cluster (:local-cluster cluster-map)]
+ (.close local-cluster)))
+
+(defmacro while-timeout [timeout-ms condition & body]
+ `(Testing/whileTimeout ~timeout-ms
+ (reify Testing$Condition (exec [this] ~condition))
+ (fn [] ~@body)))
+
+(defn wait-for-condition
+ ([apredicate]
+ (wait-for-condition Testing/TEST_TIMEOUT_MS apredicate))
+ ([timeout-ms apredicate]
+ (while-timeout timeout-ms (not (apredicate))
+ (Time/sleep 100))))
+
+(defn wait-until-cluster-waiting
+ "Wait until the cluster is idle. Should be used with time simulation."
+ ([cluster-map]
+ (let [local-cluster (:local-cluster cluster-map)]
+ (.waitForIdle local-cluster)))
+ ([cluster-map timeout-ms]
+ (let [local-cluster (:local-cluster cluster-map)]
+ (.waitForIdle local-cluster timeout-ms))))
+
+(defn advance-cluster-time
+ ([cluster-map secs increment-secs]
+ (let [local-cluster (:local-cluster cluster-map)]
+ (.advanceClusterTime local-cluster secs increment-secs)))
+ ([cluster-map secs]
+ (let [local-cluster (:local-cluster cluster-map)]
+ (.advanceClusterTime local-cluster secs))))
+
+(defmacro with-mocked-nimbus
+ [[nimbus-sym & args] & body]
+ `(let [~nimbus-sym (mk-mocked-nimbus ~@args)]
+ (try
+ ~@body
+ (catch Throwable t#
+ (log-error t# "Error in cluster")
+ (throw t#))
+ (finally
+ (let [keep-waiting?# (atom true)
+ f# (future (while @keep-waiting?# (simulate-wait ~nimbus-sym)))]
+ (kill-local-storm-cluster ~nimbus-sym)
+ (reset! keep-waiting?# false)
+ @f#)))))
+
+(defmacro with-local-cluster
+ [[cluster-sym & args] & body]
+ `(let [~cluster-sym (mk-local-storm-cluster ~@args)]
+ (try
+ ~@body
+ (catch Throwable t#
+ (log-error t# "Error in cluster")
+ (throw t#))
+ (finally
+ (let [keep-waiting?# (atom true)
+ f# (future (while @keep-waiting?# (simulate-wait ~cluster-sym)))]
+ (kill-local-storm-cluster ~cluster-sym)
+ (reset! keep-waiting?# false)
+ @f#)))))
+
+(defmacro with-simulated-time-local-cluster
+ [& args]
+ `(with-open [_# (Time$SimulatedTime.)]
+ (with-local-cluster ~@args)))
+
+(defmacro with-inprocess-zookeeper
+ [port-sym & body]
+ `(with-open [zks# (InProcessZookeeper. )]
+ (let [~port-sym (.getPort zks#)]
+ ~@body)))
+
+(defn submit-local-topology
+ [nimbus storm-name conf topology]
+ (when-not (Utils/isValidConf conf)
+ (throw (IllegalArgumentException. "Topology conf is not json-serializable")))
+ (.submitTopology nimbus storm-name nil (JSONValue/toJSONString conf) topology))
+
+(defn submit-local-topology-with-opts
+ [nimbus storm-name conf topology submit-opts]
+ (when-not (Utils/isValidConf conf)
+ (throw (IllegalArgumentException. "Topology conf is not json-serializable")))
+ (.submitTopologyWithOpts nimbus storm-name nil (JSONValue/toJSONString conf) topology submit-opts))
+
+(defn simulate-wait
+ [cluster-map]
+ (Testing/simulateWait (:local-cluster cluster-map)))
+
+(defn spout-objects [spec-map]
+ (for [[_ spout-spec] spec-map]
+ (-> spout-spec
+ .get_spout_object
+ (Thrift/deserializeComponentObject))))
+
+(defn capture-topology
+ [topology]
+ (let [cap-topo (Testing/captureTopology topology)]
+ {:topology (.topology cap-topo)
+ :capturer (.capturer cap-topo)}))
+
+(defnk complete-topology
+ [cluster-map topology
+ :mock-sources {}
+ :storm-conf {}
+ :cleanup-state true
+ :topology-name nil
+ :timeout-ms Testing/TEST_TIMEOUT_MS]
+ (Testing/completeTopology (:local-cluster cluster-map) topology,
+ (doto (CompleteTopologyParam.)
+ (.setStormConf storm-conf)
+ (.setTopologyName topology-name)
+ (.setTimeoutMs timeout-ms)
+ (.setMockedSources (MockedSources. mock-sources))
+ (.setCleanupState cleanup-state))))
+
+(defn read-tuples
+ ([results component-id stream-id]
+ (Testing/readTuples results component-id stream-id))
+ ([results component-id]
+ (Testing/readTuples results component-id )))
+
+(defn ms=
+ [a b]
+ (Testing/multiseteq a b))
+
+(def TRACKER-BOLT-ID "+++tracker-bolt")
+
+;; TODO: should override system-topology! and wrap everything there
+(defn mk-tracked-topology
+ ([tracked-cluster topology]
+ (let [tt (TrackedTopology. topology (:local-cluster tracked-cluster))]
+ {:topology (.getTopology tt)
+ :tracked-topo tt})))
+
+(defn increment-global!
+ [id key amt]
+ (-> (RegisteredGlobalState/getState id)
+ (get key)
+ (.addAndGet amt)))
+
+(defn global-amt
+ [id key]
+ (-> (RegisteredGlobalState/getState id)
+ (get key)
+ .get))
+
+(defnk mkClusterParam
+ [:supervisors 2 :ports-per-supervisor 3 :daemon-conf {} :nimbus-daemon false]
+ ;;TODO do we need to support inimbus?, group-mapper, or supervisor-slot-port-min
+ (doto (MkClusterParam. )
+ (.setDaemonConf daemon-conf)
+ (.setNimbusDaemon nimbus-daemon)
+ (.setPortsPerSupervisor (int ports-per-supervisor))
+ (.setSupervisors (int supervisors))))
+
+(defmacro with-tracked-cluster
+ [[cluster-sym & cluster-args] & body]
+ `(Testing/withTrackedCluster
+ (mkClusterParam ~@cluster-args)
+ (reify TestJob
+ (run [this# lc#]
+ (let [~cluster-sym (local-cluster-state lc#)]
+ ~@body)))))
+
+(defn tracked-wait
+ "Waits until topology is idle and 'amt' more tuples have been emitted by spouts."
+ ([tracked-map]
+ (Testing/trackedWait (:tracked-topo tracked-map)))
+ ([tracked-map amt]
+ (Testing/trackedWait (:tracked-topo tracked-map) (int amt)))
+ ([tracked-map amt timeout-ms]
+ (Testing/trackedWait (:tracked-topo tracked-map) (int amt) (int timeout-ms))))
+
+(defnk test-tuple
+ [values
+ :stream Utils/DEFAULT_STREAM_ID
+ :component "component"
+ :fields nil]
+ (Testing/testTuple
+ values
+ (doto (MkTupleParam. )
+ (.setStream stream)
+ (.setComponent component)
+ (.setFieldsList fields))))
+
+(defmacro with-timeout
+ [millis unit & body]
+ `(let [f# (future ~@body)]
+ (try
+ (.get f# ~millis ~unit)
+ (finally (future-cancel f#)))))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/LocalCluster.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/LocalCluster.clj b/storm-core/src/clj/org/apache/storm/LocalCluster.clj
deleted file mode 100644
index bce2a2e..0000000
--- a/storm-core/src/clj/org/apache/storm/LocalCluster.clj
+++ /dev/null
@@ -1,106 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-
-(ns org.apache.storm.LocalCluster
- (:use [org.apache.storm testing config])
- (:import [org.apache.storm.utils Utils])
- (:import [java.util Map])
- (:gen-class
- :init init
- :implements [org.apache.storm.ILocalCluster]
- :constructors {[] []
- [java.util.Map] []
- [String Long] []}
- :state state))
-
-(defn -init
- ([]
- (let [ret (mk-local-storm-cluster
- :daemon-conf
- {TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true})]
- [[] ret]))
- ([^String zk-host ^Long zk-port]
- (let [ret (mk-local-storm-cluster :daemon-conf {TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true
- STORM-ZOOKEEPER-SERVERS (list zk-host)
- STORM-ZOOKEEPER-PORT zk-port})]
- [[] ret]))
- ([^Map stateMap]
- [[] stateMap]))
-
-(defn submit-hook [hook name conf topology]
- (let [topologyInfo (Utils/getTopologyInfo name nil conf)]
- (.notify hook topologyInfo conf topology)))
-
-(defn -submitTopology
- [this name conf topology]
- (submit-local-topology
- (:nimbus (. this state)) name conf topology)
- (let [hook (Utils/getConfiguredClass conf STORM-TOPOLOGY-SUBMISSION-NOTIFIER-PLUGIN)]
- (when hook (submit-hook hook name conf topology))))
-
-
-(defn -submitTopologyWithOpts
- [this name conf topology submit-opts]
- (submit-local-topology-with-opts
- (:nimbus (. this state)) name conf topology submit-opts))
-
-(defn -uploadNewCredentials
- [this name creds]
- (.uploadNewCredentials (:nimbus (. this state)) name creds))
-
-(defn -shutdown
- [this]
- (kill-local-storm-cluster (. this state)))
-
-(defn -killTopology
- [this name]
- (.killTopology (:nimbus (. this state)) name))
-
-(defn -getTopologyConf
- [this id]
- (.getTopologyConf (:nimbus (. this state)) id))
-
-(defn -getTopology
- [this id]
- (.getTopology (:nimbus (. this state)) id))
-
-(defn -getClusterInfo
- [this]
- (.getClusterInfo (:nimbus (. this state))))
-
-(defn -getTopologyInfo
- [this id]
- (.getTopologyInfo (:nimbus (. this state)) id))
-
-(defn -killTopologyWithOpts
- [this name opts]
- (.killTopologyWithOpts (:nimbus (. this state)) name opts))
-
-(defn -activate
- [this name]
- (.activate (:nimbus (. this state)) name))
-
-(defn -deactivate
- [this name]
- (.deactivate (:nimbus (. this state)) name))
-
-(defn -rebalance
- [this name opts]
- (.rebalance (:nimbus (. this state)) name opts))
-
-(defn -getState
- [this]
- (.state this))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/config.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/config.clj b/storm-core/src/clj/org/apache/storm/config.clj
index d0c4d87..bfe47ed 100644
--- a/storm-core/src/clj/org/apache/storm/config.clj
+++ b/storm-core/src/clj/org/apache/storm/config.clj
@@ -15,14 +15,7 @@
;; limitations under the License.
(ns org.apache.storm.config
- (:import [java.io FileReader File IOException]
- [org.apache.storm.generated StormTopology])
- (:import [org.apache.storm Config])
- (:import [org.apache.storm.utils Utils LocalState ConfigUtils MutableInt])
- (:import [org.apache.storm.validation ConfigValidation])
- (:import [org.apache.commons.io FileUtils])
- (:require [clojure [string :as str]])
- (:use [org.apache.storm log util]))
+ (:import [org.apache.storm Config]))
(defn- clojure-config-name [name]
(.replace (.toUpperCase name) "_" "-"))
@@ -33,12 +26,3 @@
new-name (clojure-config-name name)]
(eval
`(def ~(symbol new-name) (. Config ~(symbol name))))))
-
-(def ALL-CONFIGS
- (dofor [f (seq (.getFields Config))]
- (.get f nil)))
-
-;; TODO this function and its callings will be replace when nimbus and supervisor move to Java
-(defn cluster-mode
- [conf & args]
- (keyword (conf STORM-CLUSTER-MODE)))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/converter.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/converter.clj b/storm-core/src/clj/org/apache/storm/converter.clj
deleted file mode 100644
index 6f041bb..0000000
--- a/storm-core/src/clj/org/apache/storm/converter.clj
+++ /dev/null
@@ -1,240 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.converter
- (:import [org.apache.storm.generated SupervisorInfo NodeInfo Assignment WorkerResources
- StormBase TopologyStatus ClusterWorkerHeartbeat ExecutorInfo ErrorInfo Credentials RebalanceOptions KillOptions
- TopologyActionOptions DebugOptions ProfileRequest]
- [org.apache.storm.daemon.nimbus TopologyActions]
- [org.apache.storm.utils Utils]
- [org.apache.storm.stats StatsUtil])
- (:import [org.apache.storm.cluster ExecutorBeat])
- (:use [org.apache.storm util log])
- (:require [org.apache.storm.daemon [common :as common]]))
-
-(defn thriftify-supervisor-info [supervisor-info]
- (doto (SupervisorInfo.)
- (.set_time_secs (long (:time-secs supervisor-info)))
- (.set_hostname (:hostname supervisor-info))
- (.set_assignment_id (:assignment-id supervisor-info))
- (.set_used_ports (map long (:used-ports supervisor-info)))
- (.set_meta (map long (:meta supervisor-info)))
- (.set_scheduler_meta (:scheduler-meta supervisor-info))
- (.set_uptime_secs (long (:uptime-secs supervisor-info)))
- (.set_version (:version supervisor-info))
- (.set_resources_map (:resources-map supervisor-info))
- ))
-
-(defn clojurify-supervisor-info [^SupervisorInfo supervisor-info]
- (if supervisor-info
- (org.apache.storm.daemon.common.SupervisorInfo.
- (.get_time_secs supervisor-info)
- (.get_hostname supervisor-info)
- (.get_assignment_id supervisor-info)
- (if (.get_used_ports supervisor-info) (into [] (.get_used_ports supervisor-info)))
- (if (.get_meta supervisor-info) (into [] (.get_meta supervisor-info)))
- (if (.get_scheduler_meta supervisor-info) (into {} (.get_scheduler_meta supervisor-info)))
- (.get_uptime_secs supervisor-info)
- (.get_version supervisor-info)
- (if-let [res-map (.get_resources_map supervisor-info)] (into {} res-map)))))
-
-(defn thriftify-assignment [assignment]
- (let [thrift-assignment (doto (Assignment.)
- (.set_master_code_dir (:master-code-dir assignment))
- (.set_node_host (:node->host assignment))
- (.set_executor_node_port (into {}
- (map (fn [[k v]]
- [(map long k)
- (NodeInfo. (first v) (set (map long (rest v))))])
- (:executor->node+port assignment))))
- (.set_executor_start_time_secs
- (into {}
- (map (fn [[k v]]
- [(map long k) (long v)])
- (:executor->start-time-secs assignment)))))]
- (if (:worker->resources assignment)
- (.set_worker_resources thrift-assignment (into {} (map
- (fn [[node+port resources]]
- [(NodeInfo. (first node+port) (set (map long (rest node+port))))
- (doto (WorkerResources.)
- (.set_mem_on_heap (first resources))
- (.set_mem_off_heap (second resources))
- (.set_cpu (last resources)))])
- (:worker->resources assignment)))))
- thrift-assignment))
-
-(defn clojurify-task->node_port [task->node_port]
- (into {}
- (map-val
- (fn [nodeInfo]
- (concat [(.get_node nodeInfo)] (.get_port nodeInfo))) ;nodeInfo should be converted to [node,port1,port2..]
- task->node_port)))
-
-;TODO: when translating this function, you should replace the map-key with a proper for loop HERE
-(defn clojurify-executor->node_port [executor->node_port]
- (into {}
- (map-val
- (fn [nodeInfo]
- (concat [(.get_node nodeInfo)] (.get_port nodeInfo))) ;nodeInfo should be converted to [node,port1,port2..]
- (map-key
- (fn [list-of-executors]
- (into [] list-of-executors)) ; list of executors must be coverted to clojure vector to ensure it is sortable.
- executor->node_port))))
-
-(defn thriftify-executor->node_port [executor->node_port]
- (into {}
- (map (fn [[k v]]
- [(map long k)
- (NodeInfo. (first v) (set (map long (rest v))))])
- executor->node_port))
-)
-
-(defn clojurify-worker->resources [worker->resources]
- "convert worker info to be [node, port]
- convert resources to be [mem_on_heap mem_off_heap cpu]"
- (into {} (map
- (fn [[nodeInfo resources]]
- [(concat [(.get_node nodeInfo)] (.get_port nodeInfo))
- [(.get_mem_on_heap resources) (.get_mem_off_heap resources) (.get_cpu resources)]])
- worker->resources)))
-
-;TODO: when translating this function, you should replace the map-val with a proper for loop HERE
-(defn clojurify-assignment [^Assignment assignment]
- (if assignment
- (org.apache.storm.daemon.common.Assignment.
- (.get_master_code_dir assignment)
- (into {} (.get_node_host assignment))
- (clojurify-executor->node_port (into {} (.get_executor_node_port assignment)))
- (map-key (fn [executor] (into [] executor))
- (into {} (.get_executor_start_time_secs assignment)))
- (clojurify-worker->resources (into {} (.get_worker_resources assignment))))))
-
-(defn convert-to-symbol-from-status [status]
- (if status {:type status} nil))
-
-(defn- convert-to-status-from-symbol [status]
- (if status
- (:type status)))
-
-(defn assoc-non-nil
- [m k v]
- (if v (assoc m k v) m))
-
-(defn clojurify-rebalance-options [^RebalanceOptions rebalance-options]
- (-> {:action TopologyActions/REBALANCE}
- (assoc-non-nil :delay-secs (if (.is_set_wait_secs rebalance-options) (.get_wait_secs rebalance-options)))
- (assoc-non-nil :num-workers (if (.is_set_num_workers rebalance-options) (.get_num_workers rebalance-options)))
- (assoc-non-nil :component->executors (if (.is_set_num_executors rebalance-options) (into {} (.get_num_executors rebalance-options))))))
-
-;TODO: when translating this function, you should replace the map-val with a proper for loop HERE
-(defn thriftify-rebalance-options [rebalance-options]
- (if rebalance-options
- (let [thrift-rebalance-options (RebalanceOptions.)]
- (if (:delay-secs rebalance-options)
- (.set_wait_secs thrift-rebalance-options (int (:delay-secs rebalance-options))))
- (if (:num-workers rebalance-options)
- (.set_num_workers thrift-rebalance-options (int (:num-workers rebalance-options))))
- (if (:component->executors rebalance-options)
- (.set_num_executors thrift-rebalance-options (map-val int (:component->executors rebalance-options))))
- thrift-rebalance-options)))
-
-(defn clojurify-kill-options [^KillOptions kill-options]
- (-> {:action TopologyActions/KILL}
- (assoc-non-nil :delay-secs (if (.is_set_wait_secs kill-options) (.get_wait_secs kill-options)))))
-
-(defn thriftify-kill-options [kill-options]
- (if kill-options
- (let [thrift-kill-options (KillOptions.)]
- (if (:delay-secs kill-options)
- (.set_wait_secs thrift-kill-options (int (:delay-secs kill-options))))
- thrift-kill-options)))
-
-(defn thriftify-topology-action-options [storm-base]
- (if (:topology-action-options storm-base)
- (let [ topology-action-options (:topology-action-options storm-base)
- action (:action topology-action-options)
- thrift-topology-action-options (TopologyActionOptions.)]
- (if (= action TopologyActions/KILL)
- (.set_kill_options thrift-topology-action-options (thriftify-kill-options topology-action-options)))
- (if (= action TopologyActions/REBALANCE)
- (.set_rebalance_options thrift-topology-action-options (thriftify-rebalance-options topology-action-options)))
- thrift-topology-action-options)))
-
-(defn clojurify-topology-action-options [^TopologyActionOptions topology-action-options]
- (if topology-action-options
- (or (and (.is_set_kill_options topology-action-options)
- (clojurify-kill-options
- (.get_kill_options topology-action-options)))
- (and (.is_set_rebalance_options topology-action-options)
- (clojurify-rebalance-options
- (.get_rebalance_options topology-action-options))))))
-
-(defn clojurify-debugoptions [^DebugOptions options]
- (if options
- {
- :enable (.is_enable options)
- :samplingpct (.get_samplingpct options)
- }
- ))
-
-(defn thriftify-debugoptions [options]
- (doto (DebugOptions.)
- (.set_enable (get options :enable false))
- (.set_samplingpct (get options :samplingpct 10))))
-
-;TODO: when translating this function, you should replace the map-val with a proper for loop HERE
-(defn thriftify-storm-base [storm-base]
- (doto (StormBase.)
- (.set_name (:storm-name storm-base))
- (.set_launch_time_secs (if (:launch-time-secs storm-base) (int (:launch-time-secs storm-base)) 0))
- (.set_status (convert-to-status-from-symbol (:status storm-base)))
- (.set_num_workers (if (:num-workers storm-base) (int (:num-workers storm-base)) 0))
- (.set_component_executors (map-val int (:component->executors storm-base)))
- (.set_owner (:owner storm-base))
- (.set_topology_action_options (thriftify-topology-action-options storm-base))
- (.set_prev_status (convert-to-status-from-symbol (:prev-status storm-base)))
- (.set_component_debug (map-val thriftify-debugoptions (:component->debug storm-base)))))
-
-;TODO: when translating this function, you should replace the map-val with a proper for loop HERE
-(defn clojurify-storm-base [^StormBase storm-base]
- (if storm-base
- (org.apache.storm.daemon.common.StormBase.
- (.get_name storm-base)
- (.get_launch_time_secs storm-base)
- (convert-to-symbol-from-status (.get_status storm-base))
- (.get_num_workers storm-base)
- (into {} (.get_component_executors storm-base))
- (.get_owner storm-base)
- (clojurify-topology-action-options (.get_topology_action_options storm-base))
- (convert-to-symbol-from-status (.get_prev_status storm-base))
- (map-val clojurify-debugoptions (.get_component_debug storm-base)))))
-
-(defn clojurify-profile-request
- [^ProfileRequest request]
- (when request
- {:host (.get_node (.get_nodeInfo request))
- :port (first (.get_port (.get_nodeInfo request)))
- :action (.get_action request)
- :timestamp (.get_time_stamp request)}))
-
-(defn thriftify-credentials [credentials]
- (doto (Credentials.)
- (.set_creds (if credentials credentials {}))))
-
-(defn clojurify-crdentials [^Credentials credentials]
- (if credentials
- (into {} (.get_creds credentials))
- nil
- ))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/daemon/common.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/common.clj b/storm-core/src/clj/org/apache/storm/daemon/common.clj
deleted file mode 100644
index 01a49b3..0000000
--- a/storm-core/src/clj/org/apache/storm/daemon/common.clj
+++ /dev/null
@@ -1,55 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.daemon.common
- (:use [org.apache.storm log config util])
- (:require [clojure.set :as set])
- (:import (org.apache.storm.task WorkerTopologyContext)
- (org.apache.storm.utils Utils ConfigUtils)
- (java.io InterruptedIOException)))
-
-;; the task id is the virtual port
-;; node->host is here so that tasks know who to talk to just from assignment
-;; this avoid situation where node goes down and task doesn't know what to do information-wise
-(defrecord Assignment [master-code-dir node->host executor->node+port executor->start-time-secs worker->resources])
-
-
-;; component->executors is a map from spout/bolt id to number of executors for that component
-(defrecord StormBase [storm-name launch-time-secs status num-workers component->executors owner topology-action-options prev-status component->debug])
-
-(defrecord SupervisorInfo [time-secs hostname assignment-id used-ports meta scheduler-meta uptime-secs version resources-map])
-
-(defrecord ExecutorStats [^long processed
- ^long acked
- ^long emitted
- ^long transferred
- ^long failed])
-
-(defn new-executor-stats []
- (ExecutorStats. 0 0 0 0 0))
-
-(defmacro defserverfn [name & body]
- `(let [exec-fn# (fn ~@body)]
- (defn ~name [& args#]
- (try-cause
- (apply exec-fn# args#)
- (catch InterruptedIOException e#
- (throw e#))
- (catch InterruptedException e#
- (throw e#))
- (catch Throwable t#
- (log-error t# "Error on initialization of server " ~(str name))
- (Utils/exitProcess 13 "Error on initialization")
- )))))
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
deleted file mode 100644
index a02a8e2..0000000
--- a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
+++ /dev/null
@@ -1,28 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.daemon.local-supervisor
- (:import [org.apache.storm.daemon.supervisor Supervisor]
- [org.apache.storm.utils ConfigUtils])
- (:use [org.apache.storm.daemon common])
- (:gen-class))
-
-(defserverfn mk-local-supervisor [conf shared-context isupervisor]
- (if (not (ConfigUtils/isLocalMode conf))
- (throw
- (IllegalArgumentException. "Cannot start server in distrubuted mode!")))
- (let [supervisor-server (Supervisor. conf shared-context isupervisor)]
- (.launch supervisor-server)
- supervisor-server))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/internal/thrift.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/internal/thrift.clj b/storm-core/src/clj/org/apache/storm/internal/thrift.clj
index 07e0bdf..87a0441 100644
--- a/storm-core/src/clj/org/apache/storm/internal/thrift.clj
+++ b/storm-core/src/clj/org/apache/storm/internal/thrift.clj
@@ -50,24 +50,6 @@
[^Grouping grouping]
(grouping-constants (.getSetField grouping)))
-(defn nimbus-client-and-conn
- ([host port]
- (nimbus-client-and-conn host port nil))
- ([host port as-user]
- (log-message "Connecting to Nimbus at " host ":" port " as user: " as-user)
- (let [conf (clojurify-structure (ConfigUtils/readStormConfig))
- nimbusClient (NimbusClient. conf host port nil as-user)
- client (.getClient nimbusClient)
- transport (.transport nimbusClient)]
- [client transport] )))
-
-(defmacro with-nimbus-connection
- [[client-sym host port] & body]
- `(let [[^Nimbus$Client ~client-sym ^TTransport conn#] (nimbus-client-and-conn ~host ~port)]
- (try
- ~@body
- (finally (.close conn#)))))
-
(defmacro with-configured-nimbus-connection
[client-sym & body]
`(let [conf# (clojurify-structure (ConfigUtils/readStormConfig))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/local_state_converter.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/local_state_converter.clj b/storm-core/src/clj/org/apache/storm/local_state_converter.clj
deleted file mode 100644
index e8eeaca..0000000
--- a/storm-core/src/clj/org/apache/storm/local_state_converter.clj
+++ /dev/null
@@ -1,24 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.local-state-converter
- (:import [org.apache.storm.generated ExecutorInfo]))
-
-(defn ->ExecutorInfo
- [[low high]] (ExecutorInfo. low high))
-
-(defn ->ExecutorInfo-list
- [executors]
- (map ->ExecutorInfo executors))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/log.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/log.clj b/storm-core/src/clj/org/apache/storm/log.clj
index 96570e3..7a006ef 100644
--- a/storm-core/src/clj/org/apache/storm/log.clj
+++ b/storm-core/src/clj/org/apache/storm/log.clj
@@ -15,9 +15,7 @@
;; limitations under the License.
(ns org.apache.storm.log
- (:require [clojure.tools.logging :as log])
- (:use [clojure pprint])
- (:import [java.io StringWriter]))
+ (:require [clojure.tools.logging :as log]))
(defmacro log-message
[& args]
@@ -31,26 +29,6 @@
[& args]
`(log/debug (str ~@args)))
-(defmacro log-warn-error
- [e & args]
- `(log/warn (str ~@args) ~e))
-
(defmacro log-warn
[& args]
`(log/warn (str ~@args)))
-
-(defn log-capture!
- [& args]
- (apply log/log-capture! args))
-
-(defn log-stream
- [& args]
- (apply log/log-stream args))
-
-(defmacro log-pprint
- [& args]
- `(let [^StringWriter writer# (StringWriter.)]
- (doall
- (for [object# [~@args]]
- (pprint object# writer#)))
- (log-message "\n" writer#)))
[03/10] storm git commit: STORM-1281: LocalCluster,
testing4j and testing.clj to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/cluster_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/cluster_test.clj b/storm-core/test/clj/org/apache/storm/cluster_test.clj
index 2be4a7a..583c25e 100644
--- a/storm-core/test/clj/org/apache/storm/cluster_test.clj
+++ b/storm-core/test/clj/org/apache/storm/cluster_test.clj
@@ -16,21 +16,21 @@
(ns org.apache.storm.cluster-test
(:import [java.util Arrays]
[org.apache.storm.nimbus NimbusInfo])
- (:import [org.apache.storm.daemon.common Assignment StormBase SupervisorInfo])
- (:import [org.apache.storm.generated NimbusSummary TopologyStatus])
+ (:import [org.apache.storm.generated SupervisorInfo StormBase Assignment NimbusSummary TopologyStatus NodeInfo Credentials])
(:import [org.apache.zookeeper ZooDefs ZooDefs$Ids Watcher$Event$EventType])
(:import [org.mockito Mockito])
(:import [org.mockito.exceptions.base MockitoAssertionError])
(:import [org.apache.curator.framework CuratorFramework CuratorFrameworkFactory CuratorFrameworkFactory$Builder])
- (:import [org.apache.storm.utils Time Utils ZookeeperAuthInfo ConfigUtils])
+ (:import [org.apache.storm.utils Time Time$SimulatedTime Utils ZookeeperAuthInfo ConfigUtils])
(:import [org.apache.storm.cluster IStateStorage ZKStateStorage ClusterStateContext StormClusterStateImpl ClusterUtils])
(:import [org.apache.storm.zookeeper Zookeeper])
(:import [org.apache.storm.callback ZKStateChangedCallback])
+ (:import [org.apache.storm.testing InProcessZookeeper])
(:import [org.apache.storm.testing.staticmocking MockedZookeeper MockedCluster])
(:require [conjure.core])
(:use [conjure core])
(:use [clojure test])
- (:use [org.apache.storm config util testing log converter])
+ (:use [org.apache.storm config util log])
(:use [org.apache.storm.internal thrift]))
(defn mk-config [zk-port]
@@ -53,8 +53,8 @@
(byte-array (map byte vals)))
(deftest test-basics
- (with-inprocess-zookeeper zk-port
- (let [state (mk-state zk-port)]
+ (with-open [zk (InProcessZookeeper. )]
+ (let [state (mk-state (.getPort zk))]
(.set-data state "/root" (barr 1 2 3) ZooDefs$Ids/OPEN_ACL_UNSAFE)
(is (Arrays/equals (barr 1 2 3) (.get-data state "/root" false)))
(is (= nil (.get-data state "/a" false)))
@@ -74,9 +74,9 @@
)))
(deftest test-multi-state
- (with-inprocess-zookeeper zk-port
- (let [state1 (mk-state zk-port)
- state2 (mk-state zk-port)]
+ (with-open [zk (InProcessZookeeper. )]
+ (let [state1 (mk-state (.getPort zk))
+ state2 (mk-state (.getPort zk))]
(.set-data state1 "/root" (barr 1) ZooDefs$Ids/OPEN_ACL_UNSAFE)
(is (Arrays/equals (barr 1) (.get-data state1 "/root" false)))
(is (Arrays/equals (barr 1) (.get-data state2 "/root" false)))
@@ -88,10 +88,10 @@
)))
(deftest test-ephemeral
- (with-inprocess-zookeeper zk-port
- (let [state1 (mk-state zk-port)
- state2 (mk-state zk-port)
- state3 (mk-state zk-port)]
+ (with-open [zk (InProcessZookeeper. )]
+ (let [state1 (mk-state (.getPort zk))
+ state2 (mk-state (.getPort zk))
+ state3 (mk-state (.getPort zk))]
(.set-ephemeral-node state1 "/a" (barr 1) ZooDefs$Ids/OPEN_ACL_UNSAFE)
(is (Arrays/equals (barr 1) (.get-data state1 "/a" false)))
(is (Arrays/equals (barr 1) (.get-data state2 "/a" false)))
@@ -128,11 +128,11 @@
))))
(deftest test-callbacks
- (with-inprocess-zookeeper zk-port
+ (with-open [zk (InProcessZookeeper. )]
(let [[state1-last-cb state1-cb] (mk-callback-tester)
- state1 (mk-state zk-port state1-cb)
+ state1 (mk-state (.getPort zk) state1-cb)
[state2-last-cb state2-cb] (mk-callback-tester)
- state2 (mk-state zk-port state2-cb)]
+ state2 (mk-state (.getPort zk) state2-cb)]
(.set-data state1 "/root" (barr 1) ZooDefs$Ids/OPEN_ACL_UNSAFE)
(.get-data state2 "/root" true)
(is (= nil @state1-last-cb))
@@ -178,45 +178,60 @@
)))
+(defn mkAssignment [master-code-dir node->host executor->node+port executor->start-time-secs worker->resources]
+ (doto (Assignment.)
+ (.set_executor_node_port executor->node+port)
+ (.set_executor_start_time_secs executor->start-time-secs)
+ (.set_worker_resources worker->resources)
+ (.set_node_host node->host)
+ (.set_master_code_dir master-code-dir)))
+
+(defn mkStormBase [storm-name launch-time-secs status num-workers]
+ (doto (StormBase.)
+ (.set_name storm-name)
+ (.set_launch_time_secs (int launch-time-secs))
+ (.set_status status)
+ (.set_num_workers (int num-workers))))
+
(deftest test-storm-cluster-state-basics
- (with-inprocess-zookeeper zk-port
- (let [state (mk-storm-state zk-port)
- assignment1 (Assignment. "/aaa" {} {[1] ["1" 1001 1]} {} {})
- assignment2 (Assignment. "/aaa" {} {[2] ["2" 2002]} {} {})
+ (with-open [zk (InProcessZookeeper. )]
+ (let [state (mk-storm-state (.getPort zk))
+ assignment1 (mkAssignment "/aaa" {} {[1] (NodeInfo. "1" #{1001 1})} {} {})
+ assignment2 (mkAssignment "/aaa" {} {[2] (NodeInfo. "2" #{2002})} {} {})
nimbusInfo1 (NimbusInfo. "nimbus1" 6667 false)
nimbusInfo2 (NimbusInfo. "nimbus2" 6667 false)
nimbusSummary1 (NimbusSummary. "nimbus1" 6667 (Time/currentTimeSecs) false "v1")
nimbusSummary2 (NimbusSummary. "nimbus2" 6667 (Time/currentTimeSecs) false "v2")
- base1 (StormBase. "/tmp/storm1" 1 {:type TopologyStatus/ACTIVE} 2 {} "" nil nil {})
- base2 (StormBase. "/tmp/storm2" 2 {:type TopologyStatus/ACTIVE} 2 {} "" nil nil {})]
+ base1 (mkStormBase "/tmp/storm1" 1 TopologyStatus/ACTIVE 2)
+ base2 (mkStormBase "/tmp/storm2" 2 TopologyStatus/ACTIVE 2)]
(is (= [] (.assignments state nil)))
- (.setAssignment state "storm1" (thriftify-assignment assignment1))
- (is (= assignment1 (clojurify-assignment (.assignmentInfo state "storm1" nil))))
- (is (= nil (clojurify-assignment (.assignmentInfo state "storm3" nil))))
- (.setAssignment state "storm1" (thriftify-assignment assignment2))
- (.setAssignment state "storm3" (thriftify-assignment assignment1))
+ (.setAssignment state "storm1" assignment1)
+ (is (= assignment1 (.assignmentInfo state "storm1" nil)))
+ (is (= nil (.assignmentInfo state "storm3" nil)))
+ (.setAssignment state "storm1" assignment2)
+ (.setAssignment state "storm3" assignment1)
(is (= #{"storm1" "storm3"} (set (.assignments state nil))))
- (is (= assignment2 (clojurify-assignment (.assignmentInfo state "storm1" nil))))
- (is (= assignment1 (clojurify-assignment (.assignmentInfo state "storm3" nil))))
+ (is (= assignment2 (.assignmentInfo state "storm1" nil)))
+ (is (= assignment1 (.assignmentInfo state "storm3" nil)))
(is (= [] (.activeStorms state)))
- (.activateStorm state "storm1" (thriftify-storm-base base1))
+ (.activateStorm state "storm1" base1)
(is (= ["storm1"] (.activeStorms state)))
- (is (= base1 (clojurify-storm-base (.stormBase state "storm1" nil))))
- (is (= nil (clojurify-storm-base (.stormBase state "storm2" nil))))
- (.activateStorm state "storm2" (thriftify-storm-base base2))
- (is (= base1 (clojurify-storm-base (.stormBase state "storm1" nil))))
- (is (= base2 (clojurify-storm-base (.stormBase state "storm2" nil))))
+ (is (= base1 (.stormBase state "storm1" nil)))
+ (is (= nil (.stormBase state "storm2" nil)))
+ (.activateStorm state "storm2" base2)
+ (is (= base1 (.stormBase state "storm1" nil)))
+ (is (= base2 (.stormBase state "storm2" nil)))
(is (= #{"storm1" "storm2"} (set (.activeStorms state))))
(.removeStormBase state "storm1")
- (is (= base2 (clojurify-storm-base (.stormBase state "storm2" nil))))
+ (is (= base2 (.stormBase state "storm2" nil)))
(is (= #{"storm2"} (set (.activeStorms state))))
- (is (nil? (clojurify-crdentials (.credentials state "storm1" nil))))
- (.setCredentials state "storm1" (thriftify-credentials {"a" "a"}) {})
- (is (= {"a" "a"} (clojurify-crdentials (.credentials state "storm1" nil))))
- (.setCredentials state "storm1" (thriftify-credentials {"b" "b"}) {})
- (is (= {"b" "b"} (clojurify-crdentials (.credentials state "storm1" nil))))
+ (is (nil? (.credentials state "storm1" nil)))
+ (.setCredentials state "storm1" (doto (Credentials. ) (.set_creds {"a" "a"})) {})
+ (is (= {"a" "a"} (.get_creds (.credentials state "storm1" nil))))
+ (.setCredentials state "storm1" (doto (Credentials. ) (.set_creds {"b" "b"})) {})
+ (is (= {"b" "b"} (.get_creds (.credentials state "storm1" nil))))
(is (= [] (.blobstoreInfo state "")))
(.setupBlobstore state "key1" nimbusInfo1 (Integer/parseInt "1"))
@@ -254,21 +269,21 @@
(.toString result)))
(deftest test-storm-cluster-state-errors
- (with-inprocess-zookeeper zk-port
- (with-simulated-time
- (let [state (mk-storm-state zk-port)]
+ (with-open [zk (InProcessZookeeper. )]
+ (with-open [_ (Time$SimulatedTime. )]
+ (let [state (mk-storm-state (.getPort zk))]
(.reportError state "a" "1" (Utils/localHostname) 6700 (RuntimeException.))
(validate-errors! state "a" "1" ["RuntimeException"])
- (advance-time-secs! 1)
+ (Time/advanceTimeSecs 1)
(.reportError state "a" "1" (Utils/localHostname) 6700 (IllegalArgumentException.))
(validate-errors! state "a" "1" ["IllegalArgumentException" "RuntimeException"])
(doseq [i (range 10)]
(.reportError state "a" "2" (Utils/localHostname) 6700 (RuntimeException.))
- (advance-time-secs! 2))
+ (Time/advanceTimeSecs 2))
(validate-errors! state "a" "2" (repeat 10 "RuntimeException"))
(doseq [i (range 5)]
(.reportError state "a" "2" (Utils/localHostname) 6700 (IllegalArgumentException.))
- (advance-time-secs! 2))
+ (Time/advanceTimeSecs 2))
(validate-errors! state "a" "2" (concat (repeat 5 "IllegalArgumentException")
(repeat 5 "RuntimeException")
))
@@ -276,18 +291,29 @@
(.disconnect state)
))))
+(defn mkSupervisorInfo [time-secs hostname assignment-id used-ports meta scheduler-meta uptime-secs version resources-map]
+ (doto (SupervisorInfo.)
+ (.set_time_secs time-secs)
+ (.set_hostname hostname)
+ (.set_assignment_id assignment-id)
+ (.set_used_ports used-ports)
+ (.set_meta meta)
+ (.set_scheduler_meta scheduler-meta)
+ (.set_uptime_secs uptime-secs)
+ (.set_version version)
+ (.set_resources_map resources-map)))
(deftest test-supervisor-state
- (with-inprocess-zookeeper zk-port
- (let [state1 (mk-storm-state zk-port)
- state2 (mk-storm-state zk-port)
- supervisor-info1 (SupervisorInfo. 10 "hostname-1" "id1" [1 2] [] {} 1000 "0.9.2" nil)
- supervisor-info2 (SupervisorInfo. 10 "hostname-2" "id2" [1 2] [] {} 1000 "0.9.2" nil)]
+ (with-open [zk (InProcessZookeeper. )]
+ (let [state1 (mk-storm-state (.getPort zk))
+ state2 (mk-storm-state (.getPort zk))
+ supervisor-info1 (mkSupervisorInfo 10 "hostname-1" "id1" [1 2] [] {} 1000 "0.9.2" nil)
+ supervisor-info2 (mkSupervisorInfo 10 "hostname-2" "id2" [1 2] [] {} 1000 "0.9.2" nil)]
(is (= [] (.supervisors state1 nil)))
- (.supervisorHeartbeat state2 "2" (thriftify-supervisor-info supervisor-info2))
- (.supervisorHeartbeat state1 "1" (thriftify-supervisor-info supervisor-info1))
- (is (= supervisor-info2 (clojurify-supervisor-info (.supervisorInfo state1 "2"))))
- (is (= supervisor-info1 (clojurify-supervisor-info (.supervisorInfo state1 "1"))))
+ (.supervisorHeartbeat state2 "2" supervisor-info2)
+ (.supervisorHeartbeat state1 "1" supervisor-info1)
+ (is (= supervisor-info2 (.supervisorInfo state1 "2")))
+ (is (= supervisor-info1 (.supervisorInfo state1 "1")))
(is (= #{"1" "2"} (set (.supervisors state1 nil))))
(is (= #{"1" "2"} (set (.supervisors state2 nil))))
(.disconnect state2)
@@ -296,10 +322,10 @@
)))
(deftest test-cluster-authentication
- (with-inprocess-zookeeper zk-port
+ (with-open [zk (InProcessZookeeper. )]
(let [builder (Mockito/mock CuratorFrameworkFactory$Builder)
conf (merge
- (mk-config zk-port)
+ (mk-config (.getPort zk))
{STORM-ZOOKEEPER-CONNECTION-TIMEOUT 10
STORM-ZOOKEEPER-SESSION-TIMEOUT 10
STORM-ZOOKEEPER-RETRY-INTERVAL 5
@@ -310,7 +336,7 @@
(. (Mockito/when (.connectString builder (Mockito/anyString))) (thenReturn builder))
(. (Mockito/when (.connectionTimeoutMs builder (Mockito/anyInt))) (thenReturn builder))
(. (Mockito/when (.sessionTimeoutMs builder (Mockito/anyInt))) (thenReturn builder))
- (Utils/testSetupBuilder builder (str zk-port "/") conf (ZookeeperAuthInfo. conf))
+ (Utils/testSetupBuilder builder (str (.getPort zk) "/") conf (ZookeeperAuthInfo. conf))
(is (nil?
(try
(. (Mockito/verify builder) (authorization "digest" (.getBytes (conf STORM-ZOOKEEPER-AUTH-PAYLOAD))))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/drpc_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/drpc_test.clj b/storm-core/test/clj/org/apache/storm/drpc_test.clj
index 827f536..7974e49 100644
--- a/storm-core/test/clj/org/apache/storm/drpc_test.clj
+++ b/storm-core/test/clj/org/apache/storm/drpc_test.clj
@@ -31,9 +31,9 @@
(:import [org.apache.storm Thrift])
(:import [org.apache.storm.daemon DrpcServer])
(:import [org.mockito ArgumentCaptor Mockito Matchers])
- (:use [org.apache.storm config testing])
+ (:use [org.apache.storm config])
(:use [org.apache.storm.internal clojure])
- (:use [org.apache.storm.daemon common drpc])
+ (:use [org.apache.storm.daemon drpc])
(:use [conjure core]))
(defbolt exclamation-bolt ["result" "return-info"] [tuple collector]
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/grouping_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/grouping_test.clj b/storm-core/test/clj/org/apache/storm/grouping_test.clj
index dc56f81..6eacf15 100644
--- a/storm-core/test/clj/org/apache/storm/grouping_test.clj
+++ b/storm-core/test/clj/org/apache/storm/grouping_test.clj
@@ -15,13 +15,12 @@
;; limitations under the License.
(ns org.apache.storm.grouping-test
(:use [clojure test])
- (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount TestAggregatesCounter TestWordBytesCounter NGrouping]
+ (:import [org.apache.storm.testing CompleteTopologyParam MockedSources TestWordCounter TestWordSpout TestGlobalCount TestAggregatesCounter TestWordBytesCounter NGrouping]
[org.apache.storm.generated JavaObject JavaObjectArg Grouping NullStruct])
(:import [org.apache.storm.grouping LoadMapping])
- (:use [org.apache.storm testing log config])
+ (:use [org.apache.storm log config])
(:use [org.apache.storm.internal clojure])
- (:use [org.apache.storm.daemon common])
- (:import [org.apache.storm Thrift])
+ (:import [org.apache.storm LocalCluster$Builder Testing Thrift])
(:import [org.apache.storm.utils Utils]
(org.apache.storm.daemon GrouperFactory)))
@@ -79,7 +78,9 @@
(is (<= load2 max2-prcnt))))
(deftest test-field
- (with-simulated-time-local-cluster [cluster :supervisors 4]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withSupervisors 4)))]
(let [spout-phint 4
bolt-phint 6
topology (Thrift/buildTopology
@@ -90,21 +91,24 @@
(Thrift/prepareFieldsGrouping ["word"])}
(TestWordBytesCounter.) (Integer. spout-phint))
})
- results (complete-topology
+ results (Testing/completeTopology
cluster
topology
- :mock-sources {"1" (->> [[(.getBytes "a")]
+ (doto (CompleteTopologyParam.)
+ (.setMockedSources (MockedSources. {"1" (->> [[(.getBytes "a")]
[(.getBytes "b")]]
(repeat (* spout-phint bolt-phint))
- (apply concat))})]
- (is (ms= (apply concat
+ (apply concat))}))))]
+ (is (Testing/multiseteq (apply concat
(for [value '("a" "b")
sum (range 1 (inc (* spout-phint bolt-phint)))]
- [[value sum]]))
- (read-tuples results "2"))))))
+ [[value (int sum)]]))
+ (Testing/readTuples results "2"))))))
(deftest test-field
- (with-simulated-time-local-cluster [cluster :supervisors 4]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withSupervisors 4)))]
(let [spout-phint 4
bolt-phint 6
topology (Thrift/buildTopology
@@ -115,25 +119,28 @@
(Thrift/prepareFieldsGrouping ["word"])}
(TestWordBytesCounter.) (Integer. bolt-phint))
})
- results (complete-topology
+ results (Testing/completeTopology
cluster
topology
- :mock-sources {"1" (->> [[(.getBytes "a")]
+ (doto (CompleteTopologyParam.)
+ (.setMockedSources (MockedSources. {"1" (->> [[(.getBytes "a")]
[(.getBytes "b")]]
(repeat (* spout-phint bolt-phint))
- (apply concat))})]
- (is (ms= (apply concat
+ (apply concat))}))))]
+ (is (Testing/multiseteq (apply concat
(for [value '("a" "b")
sum (range 1 (inc (* spout-phint bolt-phint)))]
- [[value sum]]))
- (read-tuples results "2"))))))
+ [[value (int sum)]]))
+ (Testing/readTuples results "2"))))))
(defbolt id-bolt ["val"] [tuple collector]
(emit-bolt! collector (.getValues tuple))
(ack! collector tuple))
(deftest test-custom-groupings
- (with-simulated-time-local-cluster [cluster]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withSupervisors 4)))]
(let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestWordSpout. true))}
@@ -150,14 +157,15 @@
id-bolt
(Integer. 6))
})
- results (complete-topology cluster
+ results (Testing/completeTopology cluster
topology
- :mock-sources {"1" [["a"]
+ (doto (CompleteTopologyParam.)
+ (.setMockedSources (MockedSources. {"1" [["a"]
["b"]
]}
- )]
- (is (ms= [["a"] ["a"] ["b"] ["b"]]
- (read-tuples results "2")))
- (is (ms= [["a"] ["a"] ["a"] ["b"] ["b"] ["b"]]
- (read-tuples results "3")))
+ ))))]
+ (is (Testing/multiseteq [["a"] ["a"] ["b"] ["b"]]
+ (Testing/readTuples results "2")))
+ (is (Testing/multiseteq [["a"] ["a"] ["a"] ["b"] ["b"] ["b"]]
+ (Testing/readTuples results "3")))
)))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/local_state_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/local_state_test.clj b/storm-core/test/clj/org/apache/storm/local_state_test.clj
index f180d69..e5baa67 100644
--- a/storm-core/test/clj/org/apache/storm/local_state_test.clj
+++ b/storm-core/test/clj/org/apache/storm/local_state_test.clj
@@ -15,15 +15,18 @@
;; limitations under the License.
(ns org.apache.storm.local-state-test
(:use [clojure test])
- (:use [org.apache.storm testing])
+ (:import [org.apache.storm.testing TmpPath])
(:import [org.apache.storm.utils LocalState]
[org.apache.storm.generated GlobalStreamId]
[org.apache.commons.io FileUtils]
[java.io File]))
(deftest test-local-state
- (with-local-tmp [dir1 dir2]
- (let [gs-a (GlobalStreamId. "a" "a")
+ (with-open [dir1-tmp (TmpPath.)
+ dir2-tmp (TmpPath.)]
+ (let [dir1 (.getPath dir1-tmp)
+ dir2 (.getPath dir2-tmp)
+ gs-a (GlobalStreamId. "a" "a")
gs-b (GlobalStreamId. "b" "b")
gs-c (GlobalStreamId. "c" "c")
gs-d (GlobalStreamId. "d" "d")
@@ -45,8 +48,9 @@
(is (= gs-d (.get ls2 "b"))))))
(deftest empty-state
- (with-local-tmp [dir]
- (let [ls (LocalState. dir)
+ (with-open [tmp-dir (TmpPath.)]
+ (let [dir (.getPath tmp-dir)
+ ls (LocalState. dir)
gs-a (GlobalStreamId. "a" "a")
data (FileUtils/openOutputStream (File. dir "12345"))
version (FileUtils/openOutputStream (File. dir "12345.version"))]
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/logviewer_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/logviewer_test.clj b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
index e9bada1..027e848 100644
--- a/storm-core/test/clj/org/apache/storm/logviewer_test.clj
+++ b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
@@ -19,8 +19,7 @@
(:require [conjure.core])
(:use [clojure test])
(:use [conjure core])
- (:use [org.apache.storm testing]
- [org.apache.storm.ui helpers])
+ (:use [org.apache.storm.ui helpers])
(:import [org.apache.storm.daemon DirectoryCleaner]
[org.apache.storm.utils Utils Time]
[org.apache.storm.utils.staticmocking UtilsInstaller]
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/messaging/netty_integration_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/messaging/netty_integration_test.clj b/storm-core/test/clj/org/apache/storm/messaging/netty_integration_test.clj
index 6a3d3ca..0e29d8c 100644
--- a/storm-core/test/clj/org/apache/storm/messaging/netty_integration_test.clj
+++ b/storm-core/test/clj/org/apache/storm/messaging/netty_integration_test.clj
@@ -17,14 +17,17 @@
(ns org.apache.storm.messaging.netty-integration-test
(:use [clojure test])
(:import [org.apache.storm.messaging TransportFactory]
- [org.apache.storm Thrift])
- (:import [org.apache.storm.testing TestWordSpout TestGlobalCount])
+ [org.apache.storm Thrift Testing LocalCluster$Builder])
+ (:import [org.apache.storm.testing CompleteTopologyParam MockedSources TestWordSpout TestGlobalCount])
(:import [org.apache.storm.utils Utils])
- (:use [org.apache.storm testing util config]))
+ (:use [org.apache.storm util config]))
(deftest test-integration
- (with-simulated-time-local-cluster [cluster :supervisors 4 :supervisor-slot-port-min 6710
- :daemon-conf {STORM-LOCAL-MODE-ZMQ true
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withSupervisors 4)
+ (.withSupervisorSlotPortMin 6710)
+ (.withDaemonConf {STORM-LOCAL-MODE-ZMQ true
STORM-MESSAGING-TRANSPORT "org.apache.storm.messaging.netty.Context"
STORM-MESSAGING-NETTY-AUTHENTICATION false
STORM-MESSAGING-NETTY-BUFFER-SIZE 1024000
@@ -32,7 +35,7 @@
STORM-MESSAGING-NETTY-MIN-SLEEP-MS 1000
STORM-MESSAGING-NETTY-MAX-SLEEP-MS 5000
STORM-MESSAGING-NETTY-CLIENT-WORKER-THREADS 1
- STORM-MESSAGING-NETTY-SERVER-WORKER-THREADS 1}]
+ STORM-MESSAGING-NETTY-SERVER-WORKER-THREADS 1})))]
(let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestWordSpout. true) (Integer. 4))}
@@ -40,12 +43,14 @@
{(Utils/getGlobalStreamId "1" nil)
(Thrift/prepareShuffleGrouping)}
(TestGlobalCount.) (Integer. 6))})
- results (complete-topology cluster
+ results (Testing/completeTopology cluster
topology
- ;; important for test that
- ;; #tuples = multiple of 4 and 6
- :storm-conf {TOPOLOGY-WORKERS 3}
- :mock-sources {"1" [["a"] ["b"]
+ (doto (CompleteTopologyParam.)
+ ;; important for test that
+ ;; #tuples = multiple of 4 and 6
+ (.setStormConf {TOPOLOGY-WORKERS 3})
+ (.setMockedSources
+ (MockedSources. {"1" [["a"] ["b"]
["a"] ["b"]
["a"] ["b"]
["a"] ["b"]
@@ -58,5 +63,5 @@
["a"] ["b"]
["a"] ["b"]
]}
- )]
- (is (= (* 6 4) (.size (read-tuples results "2")))))))
+ ))))]
+ (is (= (* 6 4) (.size (Testing/readTuples results "2")))))))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/messaging/netty_unit_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/messaging/netty_unit_test.clj b/storm-core/test/clj/org/apache/storm/messaging/netty_unit_test.clj
index 5b5914d..51a7f1c 100644
--- a/storm-core/test/clj/org/apache/storm/messaging/netty_unit_test.clj
+++ b/storm-core/test/clj/org/apache/storm/messaging/netty_unit_test.clj
@@ -17,7 +17,8 @@
(:use [clojure test])
(:import [org.apache.storm.messaging TransportFactory IConnection TaskMessage IConnectionCallback])
(:import [org.apache.storm.utils Utils])
- (:use [org.apache.storm testing util config log])
+ (:import [org.apache.storm Testing Testing$Condition])
+ (:use [org.apache.storm util config log])
(:import [java.util ArrayList]
(org.apache.storm.daemon.worker WorkerState)))
@@ -60,7 +61,9 @@
(defn- wait-for-not-nil
[atm]
- (while-timeout TEST-TIMEOUT-MS (nil? @atm) (Thread/sleep 10)))
+ (Testing/whileTimeout Testing/TEST_TIMEOUT_MS
+ (reify Testing$Condition (exec [this] (nil? @atm)))
+ (fn [] (Thread/sleep 10))))
(defn- test-basic-fn [storm-conf]
(log-message "1. Should send and receive a basic message")
@@ -112,7 +115,7 @@
_ (wait-until-ready [server client])
_ (.send client task (.getBytes req_msg))
_ (.sendLoadMetrics server {(int 1) 0.0 (int 2) 1.0})
- _ (while-timeout TEST-TIMEOUT-MS (empty? (.getLoad client [(int 1) (int 2)])) (Thread/sleep 10))
+ _ (Testing/whileTimeout Testing/TEST_TIMEOUT_MS (reify Testing$Condition (exec [this] (empty? (.getLoad client [(int 1) (int 2)])))) (fn [] (Thread/sleep 10)))
load (.getLoad client [(int 1) (int 2)])]
(is (= 0.0 (.getBoltLoad (.get load (int 1)))))
(is (= 1.0 (.getBoltLoad (.get load (int 2)))))
@@ -244,7 +247,9 @@
(let [req_msg (str num)]
(.send client task (.getBytes req_msg))))
- (while-timeout TEST-TIMEOUT-MS (< (.size resp) (- num-messages 1)) (log-message (.size resp) " " num-messages) (Thread/sleep 10))
+ (Testing/whileTimeout Testing/TEST_TIMEOUT_MS
+ (reify Testing$Condition (exec [this] (< (.size resp) (- num-messages 1))))
+ (fn [] (log-message (.size resp) " " num-messages) (Thread/sleep 10)))
(doseq [num (range 1 num-messages)]
(let [req_msg (str num)
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/messaging_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/messaging_test.clj b/storm-core/test/clj/org/apache/storm/messaging_test.clj
index 402ea7f..c984336 100644
--- a/storm-core/test/clj/org/apache/storm/messaging_test.clj
+++ b/storm-core/test/clj/org/apache/storm/messaging_test.clj
@@ -15,20 +15,22 @@
;; limitations under the License.
(ns org.apache.storm.messaging-test
(:use [clojure test])
- (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount TestEventLogSpout TestEventOrderCheckBolt])
- (:use [org.apache.storm testing config])
- (:use [org.apache.storm.daemon common])
- (:import [org.apache.storm Thrift])
+ (:import [org.apache.storm.testing CompleteTopologyParam MockedSources TestWordCounter TestWordSpout TestGlobalCount TestEventLogSpout TestEventOrderCheckBolt])
+ (:use [org.apache.storm config])
+ (:import [org.apache.storm Testing Thrift LocalCluster$Builder])
(:import [org.apache.storm.utils Utils]))
(deftest test-local-transport
- (doseq [transport-on? [false true]]
- (with-simulated-time-local-cluster [cluster :supervisors 1 :ports-per-supervisor 2
- :daemon-conf {TOPOLOGY-WORKERS 2
- STORM-LOCAL-MODE-ZMQ
- (if transport-on? true false)
- STORM-MESSAGING-TRANSPORT
- "org.apache.storm.messaging.netty.Context"}]
+ (doseq [transport-on? [false true]]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withSupervisors 1)
+ (.withPortsPerSupervisor 2)
+ (.withDaemonConf {TOPOLOGY-WORKERS 2
+ STORM-LOCAL-MODE-ZMQ
+ (if transport-on? true false)
+ STORM-MESSAGING-TRANSPORT
+ "org.apache.storm.messaging.netty.Context"})))]
(let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails
(TestWordSpout. true) (Integer. 2))}
@@ -37,11 +39,12 @@
(Thrift/prepareShuffleGrouping)}
(TestGlobalCount.) (Integer. 6))
})
- results (complete-topology cluster
+ results (Testing/completeTopology cluster
topology
- ;; important for test that
- ;; #tuples = multiple of 4 and 6
- :mock-sources {"1" [["a"] ["b"]
+ (doto (CompleteTopologyParam.)
+ ;; important for test that
+ ;; #tuples = multiple of 4 and 6
+ (.setMockedSources (MockedSources. {"1" [["a"] ["b"]
["a"] ["b"]
["a"] ["b"]
["a"] ["b"]
@@ -54,14 +57,5 @@
["a"] ["b"]
["a"] ["b"]
]}
- )]
- (is (= (* 6 4) (.size (read-tuples results "2"))))))))
-
-(extend-type TestEventLogSpout
- CompletableSpout
- (exhausted? [this]
- (-> this .completed))
- (cleanup [this]
- (.cleanup this))
- (startup [this]
- ))
+ ))))]
+ (is (= (* 6 4) (.size (Testing/readTuples results "2"))))))))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/test/clj/org/apache/storm/metrics_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/metrics_test.clj b/storm-core/test/clj/org/apache/storm/metrics_test.clj
index c993d5b..edea3ec 100644
--- a/storm-core/test/clj/org/apache/storm/metrics_test.clj
+++ b/storm-core/test/clj/org/apache/storm/metrics_test.clj
@@ -17,17 +17,17 @@
(:use [clojure test])
(:import [org.apache.storm.topology TopologyBuilder])
(:import [org.apache.storm.generated InvalidTopologyException SubmitOptions TopologyInitialStatus])
- (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount
+ (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount FeederSpout
TestAggregatesCounter TestConfBolt AckFailMapTracker PythonShellMetricsBolt PythonShellMetricsSpout])
(:import [org.apache.storm.task ShellBolt])
(:import [org.apache.storm.spout ShellSpout])
(:import [org.apache.storm.metric.api CountMetric IMetricsConsumer$DataPoint IMetricsConsumer$TaskInfo])
(:import [org.apache.storm.metric.api.rpc CountShellMetric])
(:import [org.apache.storm.utils Utils])
+ (:import [org.apache.storm Testing Testing$Condition LocalCluster$Builder])
- (:use [org.apache.storm testing config])
+ (:use [org.apache.storm config])
(:use [org.apache.storm.internal clojure])
- (:use [org.apache.storm.daemon common])
(:use [org.apache.storm.util])
(:import [org.apache.storm Thrift])
(:import [org.apache.storm.utils Utils]
@@ -50,8 +50,9 @@
))))))
(defn assert-loop [afn ids]
- (while-timeout TEST-TIMEOUT-MS (not (every? afn ids))
- (Thread/sleep 1)))
+ (Testing/whileTimeout Testing/TEST_TIMEOUT_MS
+ (reify Testing$Condition (exec [this] (not (every? afn ids))))
+ (fn [] (Thread/sleep 1))))
(defn assert-acked [tracker & ids]
(assert-loop #(.isAcked tracker %) ids))
@@ -70,16 +71,16 @@
(ack! collector tuple)))))
(defn wait-for-atleast-N-buckets! [N comp-id metric-name cluster]
- (while-timeout TEST-TIMEOUT-MS
+ (Testing/whileTimeout Testing/TEST_TIMEOUT_MS
+ (reify Testing$Condition (exec [this]
(let [taskid->buckets (clojurify-structure (FakeMetricConsumer/getTaskIdToBuckets comp-id metric-name))]
(or
(and (not= N 0) (nil? taskid->buckets))
- (not-every? #(<= N %) (map (comp count second) taskid->buckets))))
- ;;(log-message "Waiting for at least " N " timebuckets to appear in FakeMetricsConsumer for component id " comp-id " and metric name " metric-name " metrics " FakeMetricConsumer/getTaskIdToBuckets)
- (if cluster
- (advance-cluster-time cluster 1)
- (Thread/sleep 10))))
-
+ (not-every? #(<= N %) (map (comp count second) taskid->buckets))))))
+ (fn []
+ (if cluster
+ (.advanceClusterTime cluster 1)
+ (Thread/sleep 10)))))
(defn lookup-bucket-by-comp-id-&-metric-name! [comp-id metric-name]
(-> (FakeMetricConsumer/getTaskIdToBuckets comp-id metric-name)
@@ -98,65 +99,67 @@
`(is (not-empty (lookup-bucket-by-comp-id-&-metric-name! ~comp-id ~metric-name))))
(deftest test-custom-metric
- (with-simulated-time-local-cluster
- [cluster :daemon-conf {TOPOLOGY-METRICS-CONSUMER-REGISTER
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-METRICS-CONSUMER-REGISTER
[{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
"storm.zookeeper.connection.timeout" 30000
"storm.zookeeper.session.timeout" 60000
- }]
- (let [feeder (feeder-spout ["field1"])
+ })))]
+ (let [feeder (FeederSpout. ["field1"])
topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails feeder)}
{"2" (Thrift/prepareBoltDetails
{(Utils/getGlobalStreamId "1" nil)
(Thrift/prepareGlobalGrouping)}
count-acks)})]
- (submit-local-topology (:nimbus cluster) "metrics-tester" {} topology)
+ (.submitTopology cluster "metrics-tester" {} topology)
(.feed feeder ["a"] 1)
- (advance-cluster-time cluster 6)
+ (.advanceClusterTime cluster 6)
(assert-buckets! "2" "my-custom-metric" [1] cluster)
- (advance-cluster-time cluster 5)
+ (.advanceClusterTime cluster 5)
(assert-buckets! "2" "my-custom-metric" [1 0] cluster)
- (advance-cluster-time cluster 20)
+ (.advanceClusterTime cluster 20)
(assert-buckets! "2" "my-custom-metric" [1 0 0 0 0 0] cluster)
(.feed feeder ["b"] 2)
(.feed feeder ["c"] 3)
- (advance-cluster-time cluster 5)
+ (.advanceClusterTime cluster 5)
(assert-buckets! "2" "my-custom-metric" [1 0 0 0 0 0 2] cluster))))
(deftest test-custom-metric-with-multi-tasks
- (with-simulated-time-local-cluster
- [cluster :daemon-conf {TOPOLOGY-METRICS-CONSUMER-REGISTER
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-METRICS-CONSUMER-REGISTER
[{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
"storm.zookeeper.connection.timeout" 30000
"storm.zookeeper.session.timeout" 60000
- }]
- (let [feeder (feeder-spout ["field1"])
+ })))]
+ (let [feeder (FeederSpout. ["field1"])
topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails feeder)}
{"2" (Thrift/prepareBoltDetails
{(Utils/getGlobalStreamId "1" nil)
(Thrift/prepareAllGrouping)}
count-acks (Integer. 1) {TOPOLOGY-TASKS 2})})]
- (submit-local-topology (:nimbus cluster) "metrics-tester-with-multitasks" {} topology)
+ (.submitTopology cluster "metrics-tester-with-multitasks" {} topology)
(.feed feeder ["a"] 1)
- (advance-cluster-time cluster 6)
+ (.advanceClusterTime cluster 6)
(assert-buckets! "2" "my-custom-metric" [1] cluster)
- (advance-cluster-time cluster 5)
+ (.advanceClusterTime cluster 5)
(assert-buckets! "2" "my-custom-metric" [1 0] cluster)
- (advance-cluster-time cluster 20)
+ (.advanceClusterTime cluster 20)
(assert-buckets! "2" "my-custom-metric" [1 0 0 0 0 0] cluster)
(.feed feeder ["b"] 2)
(.feed feeder ["c"] 3)
- (advance-cluster-time cluster 5)
+ (.advanceClusterTime cluster 5)
(assert-buckets! "2" "my-custom-metric" [1 0 0 0 0 0 2] cluster))))
(defn mk-shell-bolt-with-metrics-spec
@@ -165,34 +168,35 @@
(PythonShellMetricsBolt. command file)))
(deftest test-custom-metric-with-multilang-py
- (with-simulated-time-local-cluster
- [cluster :daemon-conf {TOPOLOGY-METRICS-CONSUMER-REGISTER
- [{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
- "storm.zookeeper.connection.timeout" 30000
- "storm.zookeeper.session.timeout" 60000
- }]
- (let [feeder (feeder-spout ["field1"])
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-METRICS-CONSUMER-REGISTER
+ [{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
+ "storm.zookeeper.connection.timeout" 30000
+ "storm.zookeeper.session.timeout" 60000
+ })))]
+ (let [feeder (FeederSpout. ["field1"])
topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails feeder)}
{"2" (mk-shell-bolt-with-metrics-spec
{(Utils/getGlobalStreamId "1" nil)
(Thrift/prepareGlobalGrouping)}
"python" "tester_bolt_metrics.py")})]
- (submit-local-topology (:nimbus cluster) "shell-metrics-tester" {} topology)
+ (.submitTopology cluster "shell-metrics-tester" {} topology)
(.feed feeder ["a"] 1)
- (advance-cluster-time cluster 6)
+ (.advanceClusterTime cluster 6)
(assert-buckets! "2" "my-custom-shell-metric" [1] cluster)
- (advance-cluster-time cluster 5)
+ (.advanceClusterTime cluster 5)
(assert-buckets! "2" "my-custom-shell-metric" [1 0] cluster)
- (advance-cluster-time cluster 20)
+ (.advanceClusterTime cluster 20)
(assert-buckets! "2" "my-custom-shell-metric" [1 0 0 0 0 0] cluster)
(.feed feeder ["b"] 2)
(.feed feeder ["c"] 3)
- (advance-cluster-time cluster 5)
+ (.advanceClusterTime cluster 5)
(assert-buckets! "2" "my-custom-shell-metric" [1 0 0 0 0 0 2] cluster)
)))
@@ -201,48 +205,51 @@
(Thrift/prepareSpoutDetails (PythonShellMetricsSpout. command file)))
(deftest test-custom-metric-with-spout-multilang-py
- (with-simulated-time-local-cluster
- [cluster :daemon-conf {TOPOLOGY-METRICS-CONSUMER-REGISTER
- [{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
- "storm.zookeeper.connection.timeout" 30000
- "storm.zookeeper.session.timeout" 60000}]
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-METRICS-CONSUMER-REGISTER
+ [{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
+ "storm.zookeeper.connection.timeout" 30000
+ "storm.zookeeper.session.timeout" 60000
+ })))]
(let [topology (Thrift/buildTopology
{"1" (mk-shell-spout-with-metrics-spec "python" "tester_spout_metrics.py")}
{"2" (Thrift/prepareBoltDetails
{(Utils/getGlobalStreamId "1" nil)
(Thrift/prepareAllGrouping)}
count-acks)})]
- (submit-local-topology (:nimbus cluster) "shell-spout-metrics-tester" {} topology)
+ (.submitTopology cluster "shell-spout-metrics-tester" {} topology)
- (advance-cluster-time cluster 7)
+ (.advanceClusterTime cluster 7)
(assert-buckets! "1" "my-custom-shellspout-metric" [2] cluster)
)))
(deftest test-builtin-metrics-1
- (with-simulated-time-local-cluster
- [cluster :daemon-conf {TOPOLOGY-METRICS-CONSUMER-REGISTER
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-METRICS-CONSUMER-REGISTER
[{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
TOPOLOGY-STATS-SAMPLE-RATE 1.0
- TOPOLOGY-BUILTIN-METRICS-BUCKET-SIZE-SECS 60}]
- (let [feeder (feeder-spout ["field1"])
+ TOPOLOGY-BUILTIN-METRICS-BUCKET-SIZE-SECS 60})))]
+ (let [feeder (FeederSpout. ["field1"])
topology (Thrift/buildTopology
{"myspout" (Thrift/prepareSpoutDetails feeder)}
{"mybolt" (Thrift/prepareBoltDetails
{(Utils/getGlobalStreamId "myspout" nil)
(Thrift/prepareShuffleGrouping)}
acking-bolt)})]
- (submit-local-topology (:nimbus cluster) "metrics-tester" {} topology)
+ (.submitTopology cluster "metrics-tester" {} topology)
(.feed feeder ["a"] 1)
- (advance-cluster-time cluster 61)
+ (.advanceClusterTime cluster 61)
(assert-buckets! "myspout" "__ack-count/default" [1] cluster)
(assert-buckets! "myspout" "__emit-count/default" [1] cluster)
(assert-buckets! "myspout" "__transfer-count/default" [1] cluster)
(assert-buckets! "mybolt" "__ack-count/myspout:default" [1] cluster)
(assert-buckets! "mybolt" "__execute-count/myspout:default" [1] cluster)
- (advance-cluster-time cluster 120)
+ (.advanceClusterTime cluster 120)
(assert-buckets! "myspout" "__ack-count/default" [1 0 0] cluster)
(assert-buckets! "myspout" "__emit-count/default" [1 0 0] cluster)
(assert-buckets! "myspout" "__transfer-count/default" [1 0 0] cluster)
@@ -251,7 +258,7 @@
(.feed feeder ["b"] 1)
(.feed feeder ["c"] 1)
- (advance-cluster-time cluster 60)
+ (.advanceClusterTime cluster 60)
(assert-buckets! "myspout" "__ack-count/default" [1 0 0 2] cluster)
(assert-buckets! "myspout" "__emit-count/default" [1 0 0 2] cluster)
(assert-buckets! "myspout" "__transfer-count/default" [1 0 0 2] cluster)
@@ -260,12 +267,13 @@
(deftest test-builtin-metrics-2
- (with-simulated-time-local-cluster
- [cluster :daemon-conf {TOPOLOGY-METRICS-CONSUMER-REGISTER
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-METRICS-CONSUMER-REGISTER
[{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
TOPOLOGY-STATS-SAMPLE-RATE 1.0
- TOPOLOGY-BUILTIN-METRICS-BUCKET-SIZE-SECS 5}]
- (let [feeder (feeder-spout ["field1"])
+ TOPOLOGY-BUILTIN-METRICS-BUCKET-SIZE-SECS 5})))]
+ (let [feeder (FeederSpout. ["field1"])
tracker (AckFailMapTracker.)
_ (.setAckFailDelegate feeder tracker)
topology (Thrift/buildTopology
@@ -274,13 +282,13 @@
{(Utils/getGlobalStreamId "myspout" nil)
(Thrift/prepareShuffleGrouping)}
ack-every-other)})]
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"metrics-tester"
{}
topology)
(.feed feeder ["a"] 1)
- (advance-cluster-time cluster 6)
+ (.advanceClusterTime cluster 6)
(assert-buckets! "myspout" "__fail-count/default" [] cluster)
(assert-buckets! "myspout" "__ack-count/default" [1] cluster)
(assert-buckets! "myspout" "__emit-count/default" [1] cluster)
@@ -290,7 +298,7 @@
(assert-acked tracker 1)
(.feed feeder ["b"] 2)
- (advance-cluster-time cluster 5)
+ (.advanceClusterTime cluster 5)
(assert-buckets! "myspout" "__fail-count/default" [] cluster)
(assert-buckets! "myspout" "__ack-count/default" [1 0] cluster)
(assert-buckets! "myspout" "__emit-count/default" [1 1] cluster)
@@ -298,7 +306,7 @@
(assert-buckets! "mybolt" "__ack-count/myspout:default" [1 0] cluster)
(assert-buckets! "mybolt" "__execute-count/myspout:default" [1 1] cluster)
- (advance-cluster-time cluster 15)
+ (.advanceClusterTime cluster 15)
(assert-buckets! "myspout" "__ack-count/default" [1 0 0 0 0] cluster)
(assert-buckets! "myspout" "__emit-count/default" [1 1 0 0 0] cluster)
(assert-buckets! "myspout" "__transfer-count/default" [1 1 0 0 0] cluster)
@@ -306,7 +314,7 @@
(assert-buckets! "mybolt" "__execute-count/myspout:default" [1 1 0 0 0] cluster)
(.feed feeder ["c"] 3)
- (advance-cluster-time cluster 15)
+ (.advanceClusterTime cluster 15)
(assert-buckets! "myspout" "__ack-count/default" [1 0 0 0 0 1 0 0] cluster)
(assert-buckets! "myspout" "__emit-count/default" [1 1 0 0 0 1 0 0] cluster)
(assert-buckets! "myspout" "__transfer-count/default" [1 1 0 0 0 1 0 0] cluster)
@@ -314,13 +322,14 @@
(assert-buckets! "mybolt" "__execute-count/myspout:default" [1 1 0 0 0 1 0 0] cluster))))
(deftest test-builtin-metrics-3
- (with-simulated-time-local-cluster
- [cluster :daemon-conf {TOPOLOGY-METRICS-CONSUMER-REGISTER
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-METRICS-CONSUMER-REGISTER
[{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
TOPOLOGY-STATS-SAMPLE-RATE 1.0
TOPOLOGY-BUILTIN-METRICS-BUCKET-SIZE-SECS 5
- TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true}]
- (let [feeder (feeder-spout ["field1"])
+ TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS true})))]
+ (let [feeder (FeederSpout. ["field1"])
tracker (AckFailMapTracker.)
_ (.setAckFailDelegate feeder tracker)
topology (Thrift/buildTopology
@@ -329,14 +338,14 @@
{(Utils/getGlobalStreamId "myspout" nil)
(Thrift/prepareGlobalGrouping)}
ack-every-other)})]
- (submit-local-topology (:nimbus cluster)
+ (.submitTopology cluster
"timeout-tester"
{TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
topology)
(.feed feeder ["a"] 1)
(.feed feeder ["b"] 2)
(.feed feeder ["c"] 3)
- (advance-cluster-time cluster 9)
+ (.advanceClusterTime cluster 9)
(assert-buckets! "myspout" "__ack-count/default" [2] cluster)
(assert-buckets! "myspout" "__emit-count/default" [3] cluster)
(assert-buckets! "myspout" "__transfer-count/default" [3] cluster)
@@ -345,7 +354,7 @@
(assert-acked tracker 1 3)
(is (not (.isFailed tracker 2)))
- (advance-cluster-time cluster 30)
+ (.advanceClusterTime cluster 30)
(assert-failed tracker 2)
(assert-buckets! "myspout" "__fail-count/default" [1] cluster)
(assert-buckets! "myspout" "__ack-count/default" [2 0 0 0] cluster)
@@ -355,23 +364,24 @@
(assert-buckets! "mybolt" "__execute-count/myspout:default" [3 0 0 0] cluster))))
(deftest test-system-bolt
- (with-simulated-time-local-cluster
- [cluster :daemon-conf {TOPOLOGY-METRICS-CONSUMER-REGISTER
+ (with-open [cluster (.build (doto (LocalCluster$Builder.)
+ (.withSimulatedTime)
+ (.withDaemonConf {TOPOLOGY-METRICS-CONSUMER-REGISTER
[{"class" "org.apache.storm.metric.FakeMetricConsumer"}]
- TOPOLOGY-BUILTIN-METRICS-BUCKET-SIZE-SECS 60}]
- (let [feeder (feeder-spout ["field1"])
+ TOPOLOGY-BUILTIN-METRICS-BUCKET-SIZE-SECS 60})))]
+ (let [feeder (FeederSpout. ["field1"])
topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails feeder)}
{})]
- (submit-local-topology (:nimbus cluster) "metrics-tester" {} topology)
+ (.submitTopology cluster "metrics-tester" {} topology)
(.feed feeder ["a"] 1)
- (advance-cluster-time cluster 70)
+ (.advanceClusterTime cluster 70)
(assert-buckets! "__system" "newWorkerEvent" [1] cluster)
(assert-metric-data-exists! "__system" "uptimeSecs")
(assert-metric-data-exists! "__system" "startTimeSecs")
- (advance-cluster-time cluster 180)
+ (.advanceClusterTime cluster 180)
(assert-buckets! "__system" "newWorkerEvent" [1 0 0 0] cluster)
)))
[09/10] storm git commit: Merge branch 'STORM-1281' of
https://github.com/revans2/incubator-storm into STORM-1281
Posted by bo...@apache.org.
Merge branch 'STORM-1281' of https://github.com/revans2/incubator-storm into STORM-1281
STORM-1281: LocalCluster, testing4j and testing.clj to java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/557c49d5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/557c49d5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/557c49d5
Branch: refs/heads/master
Commit: 557c49d561b7278495db47a759625b45c95f6a7b
Parents: 612aea6 4efcc99
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Sat Dec 3 09:28:12 2016 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Sat Dec 3 09:28:12 2016 -0600
----------------------------------------------------------------------
.../elasticsearch/bolt/EsIndexTopology.java | 12 +-
.../trident/TridentEsTopology.java | 14 +-
.../storm/hbase/topology/LookupWordCount.java | 10 +-
.../hbase/topology/PersistentWordCount.java | 10 +-
.../storm/hbase/trident/WordCountTrident.java | 10 +-
.../storm/hdfs/bolt/HdfsFileTopology.java | 11 +-
.../storm/hdfs/bolt/SequenceFileTopology.java | 11 +-
.../storm/hdfs/trident/TridentFileTopology.java | 8 +-
.../hdfs/trident/TridentSequenceTopology.java | 8 +-
.../storm/hive/bolt/BucketTestHiveTopology.java | 12 +-
.../apache/storm/hive/bolt/HiveTopology.java | 12 +-
.../hive/bolt/HiveTopologyPartitioned.java | 12 +-
.../storm/hive/trident/TridentHiveTopology.java | 17 +-
.../jdbc/topology/AbstractUserTopology.java | 10 +-
.../storm/kafka/trident/LocalSubmitter.java | 20 +-
.../kafka/trident/TridentKafkaTopology.java | 11 +-
.../storm/mongodb/topology/InsertWordCount.java | 10 +-
.../storm/mongodb/topology/UpdateWordCount.java | 10 +-
.../storm/mongodb/trident/WordCountTrident.java | 10 +-
.../opentsdb/SampleOpenTsdbBoltTopology.java | 11 +-
.../opentsdb/SampleOpenTsdbTridentTopology.java | 11 +-
.../storm/redis/topology/LookupWordCount.java | 10 +-
.../redis/topology/PersistentWordCount.java | 10 +-
.../redis/topology/WhitelistWordCount.java | 10 +-
.../redis/trident/WordCountTridentRedis.java | 10 +-
.../trident/WordCountTridentRedisCluster.java | 10 +-
.../WordCountTridentRedisClusterMap.java | 10 +-
.../redis/trident/WordCountTridentRedisMap.java | 10 +-
.../storm/solr/topology/SolrTopology.java | 14 +-
.../apache/storm/starter/clj/exclamation.clj | 8 +-
.../storm/starter/clj/rolling_top_words.clj | 7 +-
.../org/apache/storm/starter/clj/word_count.clj | 8 +-
.../apache/storm/starter/BasicDRPCTopology.java | 17 +-
.../storm/starter/ExclamationTopology.java | 10 +-
.../org/apache/storm/starter/ManualDRPC.java | 25 +-
.../storm/starter/MultipleLoggerTopology.java | 10 +-
.../apache/storm/starter/PrintSampleStream.java | 15 +-
.../org/apache/storm/starter/ReachTopology.java | 16 +-
.../starter/ResourceAwareExampleTopology.java | 10 +-
.../apache/storm/starter/SingleJoinExample.java | 33 +-
.../storm/starter/SlidingTupleTsTopology.java | 10 +-
.../storm/starter/SlidingWindowTopology.java | 10 +-
.../storm/starter/ThroughputVsLatency.java | 2 +-
.../storm/starter/TransactionalGlobalCount.java | 13 +-
.../storm/starter/TransactionalWords.java | 15 +-
.../apache/storm/starter/WordCountTopology.java | 11 +-
.../storm/starter/WordCountTopologyNode.java | 11 +-
.../TridentHBaseWindowingStoreTopology.java | 11 +-
.../starter/trident/TridentMapExample.java | 15 +-
.../trident/TridentMinMaxOfDevicesTopology.java | 9 +-
.../TridentMinMaxOfVehiclesTopology.java | 9 +-
.../storm/starter/trident/TridentReach.java | 21 +-
.../TridentWindowingInmemoryStoreTopology.java | 10 +-
.../storm/starter/trident/TridentWordCount.java | 14 +-
.../apache/storm/starter/util/StormRunner.java | 12 +-
.../src/jvm/storm/starter/StatefulTopology.java | 11 +-
.../starter/StatefulWindowingTopology.java | 11 +-
.../main/java/org/apache/storm/flux/Flux.java | 11 +-
.../backends/trident/TestPlanCompiler.java | 32 +-
.../storm/druid/SampleDruidBoltTopology.java | 11 +-
.../druid/SampleDruidBoltTridentTopology.java | 11 +-
.../storm/eventhubs/samples/EventCount.java | 13 +-
.../storm/jms/example/ExampleJmsTopology.java | 10 +-
.../test/KafkaSpoutTopologyMainNamedTopics.java | 3 +-
.../ExponentialBackoffMsgRetryManagerTest.java | 9 +-
.../storm/mqtt/StormMqttIntegrationTest.java | 41 +-
.../org/apache/storm/ExclamationTopology.java | 10 +-
.../src/clj/org/apache/storm/clojure.clj | 18 +
.../src/clj/org/apache/storm/testing.clj | 270 ++++++
.../src/clj/org/apache/storm/LocalCluster.clj | 106 ---
storm-core/src/clj/org/apache/storm/config.clj | 18 +-
.../src/clj/org/apache/storm/converter.clj | 240 -----
.../src/clj/org/apache/storm/daemon/common.clj | 55 --
.../apache/storm/daemon/local_supervisor.clj | 28 -
.../clj/org/apache/storm/internal/thrift.clj | 18 -
.../org/apache/storm/local_state_converter.clj | 24 -
storm-core/src/clj/org/apache/storm/log.clj | 24 +-
storm-core/src/clj/org/apache/storm/testing.clj | 766 ----------------
.../src/clj/org/apache/storm/testing4j.clj | 185 ----
.../clj/org/apache/storm/trident/testing.clj | 78 --
storm-core/src/clj/org/apache/storm/ui/core.clj | 14 +-
storm-core/src/clj/org/apache/storm/util.clj | 111 ---
.../src/jvm/org/apache/storm/ILocalCluster.java | 195 +++-
.../src/jvm/org/apache/storm/ILocalDRPC.java | 13 +-
.../src/jvm/org/apache/storm/LocalCluster.java | 824 +++++++++++++++++
.../src/jvm/org/apache/storm/LocalDRPC.java | 23 +-
.../src/jvm/org/apache/storm/Testing.java | 716 +++++++++++++++
storm-core/src/jvm/org/apache/storm/Thrift.java | 2 +-
.../apache/storm/blobstore/BlobStoreUtils.java | 18 +-
.../storm/blobstore/LocalFsBlobStore.java | 8 +-
.../storm/cluster/IStormClusterState.java | 12 +-
.../org/apache/storm/daemon/nimbus/Nimbus.java | 125 +--
.../storm/daemon/supervisor/LocalContainer.java | 4 -
.../apache/storm/testing/CompletableSpout.java | 39 +
.../storm/testing/CompleteTopologyParam.java | 30 +-
.../org/apache/storm/testing/FeederSpout.java | 4 +
.../apache/storm/testing/FixedTupleSpout.java | 7 +-
.../storm/testing/InProcessZookeeper.java | 59 ++
.../apache/storm/testing/MkClusterParam.java | 6 +-
.../org/apache/storm/testing/MkTupleParam.java | 15 +-
.../apache/storm/testing/MockLeaderElector.java | 79 ++
.../org/apache/storm/testing/MockedSources.java | 8 +
.../apache/storm/testing/TestEventLogSpout.java | 7 +-
.../jvm/org/apache/storm/testing/TmpPath.java | 69 ++
.../apache/storm/testing/TrackedTopology.java | 117 ++-
.../apache/storm/topology/TopologyBuilder.java | 8 +-
.../src/jvm/org/apache/storm/utils/Time.java | 59 +-
.../org/apache/storm/integration_test.clj | 246 ++---
.../org/apache/storm/testing4j_test.clj | 20 +-
.../apache/storm/trident/integration_test.clj | 252 +++---
.../test/clj/org/apache/storm/clojure_test.clj | 159 ----
.../test/clj/org/apache/storm/cluster_test.clj | 144 +--
.../test/clj/org/apache/storm/drpc_test.clj | 4 +-
.../test/clj/org/apache/storm/grouping_test.clj | 60 +-
.../clj/org/apache/storm/local_state_test.clj | 14 +-
.../clj/org/apache/storm/logviewer_test.clj | 3 +-
.../storm/messaging/netty_integration_test.clj | 31 +-
.../apache/storm/messaging/netty_unit_test.clj | 13 +-
.../clj/org/apache/storm/messaging_test.clj | 46 +-
.../test/clj/org/apache/storm/metrics_test.clj | 170 ++--
.../test/clj/org/apache/storm/nimbus_test.clj | 891 ++++++++++---------
.../scheduler/multitenant_scheduler_test.clj | 2 +-
.../clj/org/apache/storm/scheduler_test.clj | 2 +-
.../apache/storm/security/auth/auth_test.clj | 8 +-
.../storm/security/auth/drpc_auth_test.clj | 18 +-
.../storm/security/auth/nimbus_auth_test.clj | 78 +-
.../clj/org/apache/storm/submitter_test.clj | 2 +-
.../clj/org/apache/storm/subtopology_test.clj | 62 --
.../clj/org/apache/storm/tick_tuple_test.clj | 12 +-
.../clj/org/apache/storm/transactional_test.clj | 309 +++----
.../clj/org/apache/storm/trident/state_test.clj | 2 -
.../clj/org/apache/storm/trident/tuple_test.clj | 38 +-
.../test/clj/org/apache/storm/tuple_test.clj | 13 +-
.../org/apache/storm/versioned_store_test.clj | 6 +-
.../storm/daemon/supervisor/SlotTest.java | 31 +-
.../daemon/worker/LogConfigManagerTest.java | 31 +-
.../storm/executor/error/ReportErrorTest.java | 6 +-
.../jvm/org/apache/storm/utils/TimeTest.java | 31 +-
138 files changed, 4274 insertions(+), 3661 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/557c49d5/storm-core/src/jvm/org/apache/storm/daemon/nimbus/Nimbus.java
----------------------------------------------------------------------
[08/10] storm git commit: STORM-1281: LocalCluster,
testing4j and testing.clj to java
Posted by bo...@apache.org.
STORM-1281: LocalCluster, testing4j and testing.clj to java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4efcc996
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4efcc996
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4efcc996
Branch: refs/heads/master
Commit: 4efcc996aad0e30469b310d74efd58f0c0249d91
Parents: 2832aeb
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Oct 24 15:59:36 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Dec 1 14:15:22 2016 -0600
----------------------------------------------------------------------
.../elasticsearch/bolt/EsIndexTopology.java | 12 +-
.../trident/TridentEsTopology.java | 14 +-
.../storm/hbase/topology/LookupWordCount.java | 10 +-
.../hbase/topology/PersistentWordCount.java | 10 +-
.../storm/hbase/trident/WordCountTrident.java | 10 +-
.../storm/hdfs/bolt/HdfsFileTopology.java | 11 +-
.../storm/hdfs/bolt/SequenceFileTopology.java | 11 +-
.../storm/hdfs/trident/TridentFileTopology.java | 8 +-
.../hdfs/trident/TridentSequenceTopology.java | 8 +-
.../storm/hive/bolt/BucketTestHiveTopology.java | 12 +-
.../apache/storm/hive/bolt/HiveTopology.java | 12 +-
.../hive/bolt/HiveTopologyPartitioned.java | 12 +-
.../storm/hive/trident/TridentHiveTopology.java | 17 +-
.../jdbc/topology/AbstractUserTopology.java | 10 +-
.../storm/kafka/trident/LocalSubmitter.java | 20 +-
.../kafka/trident/TridentKafkaTopology.java | 11 +-
.../storm/mongodb/topology/InsertWordCount.java | 10 +-
.../storm/mongodb/topology/UpdateWordCount.java | 10 +-
.../storm/mongodb/trident/WordCountTrident.java | 10 +-
.../opentsdb/SampleOpenTsdbBoltTopology.java | 11 +-
.../opentsdb/SampleOpenTsdbTridentTopology.java | 11 +-
.../storm/redis/topology/LookupWordCount.java | 10 +-
.../redis/topology/PersistentWordCount.java | 10 +-
.../redis/topology/WhitelistWordCount.java | 10 +-
.../redis/trident/WordCountTridentRedis.java | 10 +-
.../trident/WordCountTridentRedisCluster.java | 10 +-
.../WordCountTridentRedisClusterMap.java | 10 +-
.../redis/trident/WordCountTridentRedisMap.java | 10 +-
.../storm/solr/topology/SolrTopology.java | 14 +-
.../apache/storm/starter/clj/exclamation.clj | 8 +-
.../storm/starter/clj/rolling_top_words.clj | 7 +-
.../org/apache/storm/starter/clj/word_count.clj | 8 +-
.../apache/storm/starter/BasicDRPCTopology.java | 17 +-
.../storm/starter/ExclamationTopology.java | 10 +-
.../org/apache/storm/starter/ManualDRPC.java | 25 +-
.../storm/starter/MultipleLoggerTopology.java | 10 +-
.../apache/storm/starter/PrintSampleStream.java | 15 +-
.../org/apache/storm/starter/ReachTopology.java | 16 +-
.../starter/ResourceAwareExampleTopology.java | 10 +-
.../apache/storm/starter/SingleJoinExample.java | 33 +-
.../storm/starter/SlidingTupleTsTopology.java | 10 +-
.../storm/starter/SlidingWindowTopology.java | 10 +-
.../storm/starter/ThroughputVsLatency.java | 2 +-
.../storm/starter/TransactionalGlobalCount.java | 13 +-
.../storm/starter/TransactionalWords.java | 15 +-
.../apache/storm/starter/WordCountTopology.java | 11 +-
.../storm/starter/WordCountTopologyNode.java | 11 +-
.../TridentHBaseWindowingStoreTopology.java | 11 +-
.../starter/trident/TridentMapExample.java | 15 +-
.../trident/TridentMinMaxOfDevicesTopology.java | 9 +-
.../TridentMinMaxOfVehiclesTopology.java | 9 +-
.../storm/starter/trident/TridentReach.java | 21 +-
.../TridentWindowingInmemoryStoreTopology.java | 10 +-
.../storm/starter/trident/TridentWordCount.java | 14 +-
.../apache/storm/starter/util/StormRunner.java | 12 +-
.../src/jvm/storm/starter/StatefulTopology.java | 11 +-
.../starter/StatefulWindowingTopology.java | 11 +-
.../main/java/org/apache/storm/flux/Flux.java | 11 +-
.../backends/trident/TestPlanCompiler.java | 32 +-
.../storm/druid/SampleDruidBoltTopology.java | 11 +-
.../druid/SampleDruidBoltTridentTopology.java | 11 +-
.../storm/eventhubs/samples/EventCount.java | 13 +-
.../storm/jms/example/ExampleJmsTopology.java | 10 +-
.../test/KafkaSpoutTopologyMainNamedTopics.java | 3 +-
.../ExponentialBackoffMsgRetryManagerTest.java | 9 +-
.../storm/mqtt/StormMqttIntegrationTest.java | 41 +-
.../org/apache/storm/ExclamationTopology.java | 10 +-
.../src/clj/org/apache/storm/clojure.clj | 18 +
.../src/clj/org/apache/storm/testing.clj | 270 ++++++
.../src/clj/org/apache/storm/LocalCluster.clj | 106 ---
storm-core/src/clj/org/apache/storm/config.clj | 18 +-
.../src/clj/org/apache/storm/converter.clj | 240 -----
.../src/clj/org/apache/storm/daemon/common.clj | 55 --
.../apache/storm/daemon/local_supervisor.clj | 28 -
.../clj/org/apache/storm/internal/thrift.clj | 18 -
.../org/apache/storm/local_state_converter.clj | 24 -
storm-core/src/clj/org/apache/storm/log.clj | 24 +-
storm-core/src/clj/org/apache/storm/testing.clj | 766 ----------------
.../src/clj/org/apache/storm/testing4j.clj | 185 ----
.../clj/org/apache/storm/trident/testing.clj | 78 --
storm-core/src/clj/org/apache/storm/ui/core.clj | 14 +-
storm-core/src/clj/org/apache/storm/util.clj | 111 ---
.../src/jvm/org/apache/storm/ILocalCluster.java | 195 +++-
.../src/jvm/org/apache/storm/ILocalDRPC.java | 13 +-
.../src/jvm/org/apache/storm/LocalCluster.java | 824 +++++++++++++++++
.../src/jvm/org/apache/storm/LocalDRPC.java | 23 +-
.../src/jvm/org/apache/storm/Testing.java | 716 +++++++++++++++
storm-core/src/jvm/org/apache/storm/Thrift.java | 2 +-
.../apache/storm/blobstore/BlobStoreUtils.java | 18 +-
.../storm/blobstore/LocalFsBlobStore.java | 8 +-
.../storm/cluster/IStormClusterState.java | 12 +-
.../org/apache/storm/daemon/nimbus/Nimbus.java | 125 +--
.../storm/daemon/supervisor/LocalContainer.java | 4 -
.../apache/storm/testing/CompletableSpout.java | 39 +
.../storm/testing/CompleteTopologyParam.java | 30 +-
.../org/apache/storm/testing/FeederSpout.java | 4 +
.../apache/storm/testing/FixedTupleSpout.java | 7 +-
.../storm/testing/InProcessZookeeper.java | 59 ++
.../apache/storm/testing/MkClusterParam.java | 6 +-
.../org/apache/storm/testing/MkTupleParam.java | 15 +-
.../apache/storm/testing/MockLeaderElector.java | 79 ++
.../org/apache/storm/testing/MockedSources.java | 8 +
.../apache/storm/testing/TestEventLogSpout.java | 7 +-
.../jvm/org/apache/storm/testing/TmpPath.java | 69 ++
.../apache/storm/testing/TrackedTopology.java | 117 ++-
.../apache/storm/topology/TopologyBuilder.java | 8 +-
.../src/jvm/org/apache/storm/utils/Time.java | 59 +-
.../org/apache/storm/integration_test.clj | 246 ++---
.../org/apache/storm/testing4j_test.clj | 20 +-
.../apache/storm/trident/integration_test.clj | 252 +++---
.../test/clj/org/apache/storm/clojure_test.clj | 159 ----
.../test/clj/org/apache/storm/cluster_test.clj | 144 +--
.../test/clj/org/apache/storm/drpc_test.clj | 4 +-
.../test/clj/org/apache/storm/grouping_test.clj | 60 +-
.../clj/org/apache/storm/local_state_test.clj | 14 +-
.../clj/org/apache/storm/logviewer_test.clj | 3 +-
.../storm/messaging/netty_integration_test.clj | 31 +-
.../apache/storm/messaging/netty_unit_test.clj | 13 +-
.../clj/org/apache/storm/messaging_test.clj | 46 +-
.../test/clj/org/apache/storm/metrics_test.clj | 170 ++--
.../test/clj/org/apache/storm/nimbus_test.clj | 891 ++++++++++---------
.../scheduler/multitenant_scheduler_test.clj | 2 +-
.../clj/org/apache/storm/scheduler_test.clj | 2 +-
.../apache/storm/security/auth/auth_test.clj | 8 +-
.../storm/security/auth/drpc_auth_test.clj | 18 +-
.../storm/security/auth/nimbus_auth_test.clj | 78 +-
.../clj/org/apache/storm/submitter_test.clj | 2 +-
.../clj/org/apache/storm/subtopology_test.clj | 62 --
.../clj/org/apache/storm/tick_tuple_test.clj | 12 +-
.../clj/org/apache/storm/transactional_test.clj | 309 +++----
.../clj/org/apache/storm/trident/state_test.clj | 2 -
.../clj/org/apache/storm/trident/tuple_test.clj | 38 +-
.../test/clj/org/apache/storm/tuple_test.clj | 13 +-
.../org/apache/storm/versioned_store_test.clj | 6 +-
.../storm/daemon/supervisor/SlotTest.java | 31 +-
.../daemon/worker/LogConfigManagerTest.java | 31 +-
.../storm/executor/error/ReportErrorTest.java | 6 +-
.../jvm/org/apache/storm/utils/TimeTest.java | 31 +-
138 files changed, 4274 insertions(+), 3661 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/bolt/EsIndexTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/bolt/EsIndexTopology.java b/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/bolt/EsIndexTopology.java
index d30424b..0a518cd 100644
--- a/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/bolt/EsIndexTopology.java
+++ b/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/bolt/EsIndexTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.elasticsearch.bolt;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -54,13 +55,10 @@ public class EsIndexTopology {
EsTestUtil.startEsNode();
EsTestUtil.waitForSeconds(5);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
- EsTestUtil.waitForSeconds(20);
- cluster.killTopology(TOPOLOGY_NAME);
- System.out.println("cluster begin to shutdown");
- cluster.shutdown();
- System.out.println("cluster shutdown");
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());) {
+ EsTestUtil.waitForSeconds(20);
+ }
System.exit(0);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/trident/TridentEsTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/trident/TridentEsTopology.java b/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/trident/TridentEsTopology.java
index 67eab5b..4293c8c 100644
--- a/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/trident/TridentEsTopology.java
+++ b/examples/storm-elasticsearch-examples/src/main/java/org/apache/storm/elasticsearch/trident/TridentEsTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.elasticsearch.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
@@ -39,7 +40,7 @@ public class TridentEsTopology {
static final String TOPOLOGY_NAME = "elasticsearch-test-topology2";
- public static void main(String[] args) {
+ public static void main(String[] args) throws Exception {
int batchSize = 100;
FixedBatchSpout spout = new FixedBatchSpout(batchSize);
spout.setCycle(true);
@@ -55,13 +56,10 @@ public class TridentEsTopology {
EsTestUtil.startEsNode();
EsTestUtil.waitForSeconds(5);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology(TOPOLOGY_NAME, null, topology.build());
- EsTestUtil.waitForSeconds(20);
- cluster.killTopology(TOPOLOGY_NAME);
- System.out.println("cluster begin to shutdown");
- cluster.shutdown();
- System.out.println("cluster shutdown");
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, null, topology.build());) {
+ EsTestUtil.waitForSeconds(20);
+ }
System.exit(0);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java
index 43f72ae..afbfafd 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java
@@ -19,6 +19,7 @@ package org.apache.storm.hbase.topology;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
@@ -64,11 +65,10 @@ public class LookupWordCount {
builder.setBolt(TOTAL_COUNT_BOLT, totalBolt, 1).fieldsGrouping(LOOKUP_BOLT, new Fields("columnName"));
if (args.length == 1) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", config, builder.createTopology());
- Thread.sleep(30000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
} else if (args.length == 2) {
StormSubmitter.submitTopology(args[1], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java
index cfb94d0..ed866e9 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java
@@ -20,6 +20,7 @@ package org.apache.storm.hbase.topology;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
@@ -69,11 +70,10 @@ public class PersistentWordCount {
if (args.length == 1) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", config, builder.createTopology());
- Thread.sleep(30000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
} else if (args.length == 2) {
StormSubmitter.submitTopology(args[1], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java
index b2f0ce8..62a2005 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java
@@ -19,6 +19,7 @@ package org.apache.storm.hbase.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.tuple.Fields;
@@ -86,11 +87,10 @@ public class WordCountTrident {
Config conf = new Config();
conf.setMaxSpoutPending(5);
if (args.length == 1) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
- Thread.sleep(60 * 1000);
- cluster.killTopology("wordCounter");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));) {
+ Thread.sleep(60 * 1000);
+ }
System.exit(0);
}
else if(args.length == 2) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
index b1ae542..a5c2fe3 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.hdfs.bolt;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.OutputCollector;
@@ -98,12 +99,10 @@ public class HdfsFileTopology {
.shuffleGrouping(SENTENCE_SPOUT_ID);
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
-
- cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
- waitForSeconds(120);
- cluster.killTopology(TOPOLOGY_NAME);
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());) {
+ waitForSeconds(120);
+ }
System.exit(0);
} else if (args.length == 3) {
StormSubmitter.submitTopology(args[2], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
index 86bc698..abe1ebd 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.hdfs.bolt;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.OutputCollector;
@@ -98,12 +99,10 @@ public class SequenceFileTopology {
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
-
- cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
- waitForSeconds(120);
- cluster.killTopology(TOPOLOGY_NAME);
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());) {
+ waitForSeconds(120);
+ }
System.exit(0);
} else if(args.length == 3) {
StormSubmitter.submitTopology(args[2], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
index 8f75c45..1e830a1 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.hdfs.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.tuple.Fields;
@@ -86,9 +87,10 @@ public class TridentFileTopology {
conf.put("hdfs.config", yamlConf);
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
- Thread.sleep(120 * 1000);
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));) {
+ Thread.sleep(120 * 1000);
+ }
} else if(args.length == 3) {
conf.setNumWorkers(3);
StormSubmitter.submitTopology(args[2], conf, buildTopology(args[0]));
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
index 788b33c..3248363 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.hdfs.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.tuple.Fields;
@@ -83,9 +84,10 @@ public class TridentSequenceTopology {
conf.put("hdfs.config", yamlConf);
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
- Thread.sleep(120 * 1000);
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));) {
+ Thread.sleep(120 * 1000);
+ }
} else if(args.length == 3) {
conf.setNumWorkers(3);
StormSubmitter.submitTopology(args[2], conf, buildTopology(args[0]));
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/BucketTestHiveTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/BucketTestHiveTopology.java b/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/BucketTestHiveTopology.java
index 781a539..e80a118 100644
--- a/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/BucketTestHiveTopology.java
+++ b/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/BucketTestHiveTopology.java
@@ -20,6 +20,7 @@ package org.apache.storm.hive.bolt;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -94,13 +95,10 @@ public class BucketTestHiveTopology {
builder.setBolt(BOLT_ID, hiveBolt, 14)
.shuffleGrouping(USER_SPOUT_ID);
if (args.length == 6) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
- waitForSeconds(20);
- cluster.killTopology(TOPOLOGY_NAME);
- System.out.println("cluster begin to shutdown");
- cluster.shutdown();
- System.out.println("cluster shutdown");
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());) {
+ waitForSeconds(20);
+ }
System.exit(0);
} else {
StormSubmitter.submitTopology(args[7], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopology.java b/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopology.java
index 4afd298..e69c68f 100644
--- a/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopology.java
+++ b/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopology.java
@@ -20,6 +20,7 @@ package org.apache.storm.hive.bolt;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -75,13 +76,10 @@ public class HiveTopology {
builder.setBolt(BOLT_ID, hiveBolt, 1)
.shuffleGrouping(USER_SPOUT_ID);
if (args.length == 3) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
- waitForSeconds(20);
- cluster.killTopology(TOPOLOGY_NAME);
- System.out.println("cluster begin to shutdown");
- cluster.shutdown();
- System.out.println("cluster shutdown");
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());) {
+ waitForSeconds(20);
+ }
System.exit(0);
} else if(args.length >= 4) {
StormSubmitter.submitTopology(args[3], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopologyPartitioned.java
----------------------------------------------------------------------
diff --git a/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopologyPartitioned.java b/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopologyPartitioned.java
index a52c490..f64e626 100644
--- a/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopologyPartitioned.java
+++ b/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/bolt/HiveTopologyPartitioned.java
@@ -20,6 +20,7 @@ package org.apache.storm.hive.bolt;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -77,13 +78,10 @@ public class HiveTopologyPartitioned {
builder.setBolt(BOLT_ID, hiveBolt, 1)
.shuffleGrouping(USER_SPOUT_ID);
if (args.length == 3) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
- waitForSeconds(20);
- cluster.killTopology(TOPOLOGY_NAME);
- System.out.println("cluster begin to shutdown");
- cluster.shutdown();
- System.out.println("cluster shutdown");
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());) {
+ waitForSeconds(20);
+ }
System.exit(0);
} else if(args.length >= 4) {
StormSubmitter.submitTopology(args[3], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/trident/TridentHiveTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/trident/TridentHiveTopology.java b/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/trident/TridentHiveTopology.java
index 86a35e6..4505561 100644
--- a/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/trident/TridentHiveTopology.java
+++ b/examples/storm-hive-examples/src/main/java/org/apache/storm/hive/trident/TridentHiveTopology.java
@@ -24,6 +24,7 @@ import org.apache.storm.hive.common.HiveOptions;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.hooks.SubmitterHookException;
@@ -89,22 +90,18 @@ public class TridentHiveTopology {
}
}
- public static void main(String[] args) {
+ public static void main(String[] args) throws Exception {
String metaStoreURI = args[0];
String dbName = args[1];
String tblName = args[2];
Config conf = new Config();
conf.setMaxSpoutPending(5);
if(args.length == 3) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("tridentHiveTopology", conf, buildTopology(metaStoreURI, dbName, tblName,null,null));
- LOG.info("waiting for 60 seconds");
- waitForSeconds(60);
- LOG.info("killing topology");
- cluster.killTopology("tridenHiveTopology");
- LOG.info("cluster shutdown");
- cluster.shutdown();
- LOG.info("cluster shutdown");
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("tridentHiveTopology", conf, buildTopology(metaStoreURI, dbName, tblName,null,null));) {
+ LOG.info("waiting for 60 seconds");
+ waitForSeconds(60);
+ }
System.exit(0);
} else if(args.length == 4) {
try {
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java
index ec7ca36..19cdf74 100644
--- a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java
+++ b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java
@@ -33,6 +33,7 @@ import org.apache.storm.jdbc.mapper.SimpleJdbcMapper;
import org.apache.storm.jdbc.mapper.SimpleJdbcLookupMapper;
import org.apache.storm.jdbc.spout.UserSpout;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import java.sql.Types;
import java.util.List;
@@ -99,11 +100,10 @@ public abstract class AbstractUserTopology {
this.jdbcLookupMapper = new SimpleJdbcLookupMapper(outputFields, queryParamColumns);
this.connectionProvider = new HikariCPConnectionProvider(map);
if (args.length == 4) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", config, getTopology());
- Thread.sleep(30000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", config, getTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
} else {
StormSubmitter.submitTopology(args[4], config, getTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/LocalSubmitter.java
----------------------------------------------------------------------
diff --git a/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/LocalSubmitter.java b/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/LocalSubmitter.java
index a47c67e..54ec99c 100644
--- a/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/LocalSubmitter.java
+++ b/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/LocalSubmitter.java
@@ -38,8 +38,12 @@ public class LocalSubmitter {
this.cluster = cluster;
}
- public static LocalSubmitter newInstance() {
- return new LocalSubmitter(new LocalDRPC(), new LocalCluster());
+ public static LocalSubmitter newInstance() {
+ try {
+ return new LocalSubmitter(new LocalDRPC(), new LocalCluster());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
public static Config defaultConfig() {
@@ -55,7 +59,11 @@ public class LocalSubmitter {
}
public void submit(String name, Config config, StormTopology topology) {
- cluster.submitTopology(name, config, topology);
+ try {
+ cluster.submitTopology(name, config, topology);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
/**
@@ -74,7 +82,11 @@ public class LocalSubmitter {
}
public void kill (String name) {
- cluster.killTopology(name);
+ try {
+ cluster.killTopology(name);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
public void shutdown() {
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaTopology.java b/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaTopology.java
index babb502..11ea899 100644
--- a/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaTopology.java
+++ b/examples/storm-kafka-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaTopology.java
@@ -20,6 +20,7 @@ package org.apache.storm.kafka.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.kafka.trident.mapper.FieldNameBasedTupleToKafkaMapper;
import org.apache.storm.kafka.trident.selector.DefaultTopicSelector;
@@ -79,11 +80,9 @@ public class TridentKafkaTopology {
System.out.println("Please provide kafka broker url ,e.g. localhost:9092");
}
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("wordCounter", new Config(), buildTopology(args[0]));
- Thread.sleep(60 * 1000);
- cluster.killTopology("wordCounter");
-
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("wordCounter", new Config(), buildTopology(args[0]));) {
+ Thread.sleep(60 * 1000);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/InsertWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/InsertWordCount.java b/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/InsertWordCount.java
index 366acf4..6f71b1c 100644
--- a/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/InsertWordCount.java
+++ b/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/InsertWordCount.java
@@ -19,6 +19,7 @@ package org.apache.storm.mongodb.topology;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
@@ -66,11 +67,10 @@ public class InsertWordCount {
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", config, builder.createTopology());
- Thread.sleep(30000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
} else if (args.length == 3) {
StormSubmitter.submitTopology(args[2], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/UpdateWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/UpdateWordCount.java b/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/UpdateWordCount.java
index 7895f35..f80cc7c 100644
--- a/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/UpdateWordCount.java
+++ b/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/topology/UpdateWordCount.java
@@ -19,6 +19,7 @@ package org.apache.storm.mongodb.topology;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
@@ -76,11 +77,10 @@ public class UpdateWordCount {
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", config, builder.createTopology());
- Thread.sleep(30000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
} else if (args.length == 3) {
StormSubmitter.submitTopology(args[2], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/trident/WordCountTrident.java
----------------------------------------------------------------------
diff --git a/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/trident/WordCountTrident.java b/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/trident/WordCountTrident.java
index 44447be..ae97961 100644
--- a/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/trident/WordCountTrident.java
+++ b/examples/storm-mongodb-examples/src/main/java/org/apache/storm/mongodb/trident/WordCountTrident.java
@@ -19,6 +19,7 @@ package org.apache.storm.mongodb.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.mongodb.common.mapper.MongoMapper;
@@ -67,11 +68,10 @@ public class WordCountTrident {
Config conf = new Config();
conf.setMaxSpoutPending(5);
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("wordCounter", conf, buildTopology(args[0], args[1]));
- Thread.sleep(60 * 1000);
- cluster.killTopology("wordCounter");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(args[0], args[1]));) {
+ Thread.sleep(60 * 1000);
+ }
System.exit(0);
}
else if(args.length == 3) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbBoltTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbBoltTopology.java b/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbBoltTopology.java
index 6c511b8..7dc65fd 100644
--- a/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbBoltTopology.java
+++ b/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbBoltTopology.java
@@ -23,6 +23,7 @@ import org.apache.storm.opentsdb.bolt.TupleOpenTsdbDatapointMapper;
import org.apache.storm.opentsdb.client.OpenTsdbClient;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
@@ -58,12 +59,10 @@ public class SampleOpenTsdbBoltTopology {
} else {
conf.setMaxTaskParallelism(3);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("word-count", conf, topologyBuilder.createTopology());
-
- Thread.sleep(30000);
-
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("word-count", conf, topologyBuilder.createTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbTridentTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbTridentTopology.java b/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbTridentTopology.java
index db51a8a..3220068 100644
--- a/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbTridentTopology.java
+++ b/examples/storm-opentsdb-examples/src/main/java/org/apache/storm/opentsdb/SampleOpenTsdbTridentTopology.java
@@ -24,6 +24,7 @@ import org.apache.storm.opentsdb.trident.OpenTsdbStateFactory;
import org.apache.storm.opentsdb.trident.OpenTsdbStateUpdater;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.trident.Stream;
import org.apache.storm.trident.TridentTopology;
@@ -74,12 +75,10 @@ public class SampleOpenTsdbTridentTopology {
} else {
conf.setMaxTaskParallelism(3);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("word-count", conf, tridentTopology.build());
-
- Thread.sleep(30000);
-
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("word-count", conf, tridentTopology.build())) {
+ Thread.sleep(30000);
+ }
System.exit(0);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java
index f62b7b0..17a088b 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java
@@ -19,6 +19,7 @@ package org.apache.storm.redis.topology;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -109,11 +110,10 @@ public class LookupWordCount {
builder.setBolt(PRINT_BOLT, printBolt, 1).shuffleGrouping(LOOKUP_BOLT);
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", config, builder.createTopology());
- Thread.sleep(30000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
} else if (args.length == 3) {
StormSubmitter.submitTopology(args[2], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java
index d46bab6..36cec89 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java
@@ -19,6 +19,7 @@ package org.apache.storm.redis.topology;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.TopologyBuilder;
@@ -72,11 +73,10 @@ public class PersistentWordCount {
builder.setBolt(STORE_BOLT, storeBolt, 1).shuffleGrouping(COUNT_BOLT);
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", config, builder.createTopology());
- Thread.sleep(30000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
} else if (args.length == 3) {
StormSubmitter.submitTopology(args[2], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java
index bcb2e0b..1cccd9b 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java
@@ -19,6 +19,7 @@ package org.apache.storm.redis.topology;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.redis.bolt.RedisFilterBolt;
import org.apache.storm.redis.common.config.JedisPoolConfig;
@@ -106,11 +107,10 @@ public class WhitelistWordCount {
builder.setBolt(PRINT_BOLT, printBolt, 1).shuffleGrouping(COUNT_BOLT);
if (args.length == 2) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", config, builder.createTopology());
- Thread.sleep(30000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology());) {
+ Thread.sleep(30000);
+ }
System.exit(0);
} else if (args.length == 3) {
StormSubmitter.submitTopology(args[2], config, builder.createTopology());
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java
----------------------------------------------------------------------
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java
index e3eb0f9..f5278fd 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java
@@ -19,6 +19,7 @@ package org.apache.storm.redis.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.tuple.Fields;
@@ -82,11 +83,10 @@ public class WordCountTridentRedis {
Config conf = new Config();
conf.setMaxSpoutPending(5);
if (flag == 0) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, redisPort));
- Thread.sleep(60 * 1000);
- cluster.killTopology("test_wordCounter_for_redis");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, redisPort));) {
+ Thread.sleep(60 * 1000);
+ }
System.exit(0);
} else if(flag == 1) {
conf.setNumWorkers(3);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java
----------------------------------------------------------------------
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java
index 116a58a..b6e067d 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java
@@ -19,6 +19,7 @@ package org.apache.storm.redis.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.tuple.Fields;
@@ -89,11 +90,10 @@ public class WordCountTridentRedisCluster {
Config conf = new Config();
conf.setMaxSpoutPending(5);
if (flag == 0) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort));
- Thread.sleep(60 * 1000);
- cluster.killTopology("test_wordCounter_for_redis");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort));) {
+ Thread.sleep(60 * 1000);
+ }
System.exit(0);
} else if(flag == 1) {
conf.setNumWorkers(3);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java
----------------------------------------------------------------------
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java
index fafb4e0..0a025bd 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java
@@ -19,6 +19,7 @@ package org.apache.storm.redis.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.tuple.Fields;
@@ -84,11 +85,10 @@ public class WordCountTridentRedisClusterMap {
Config conf = new Config();
conf.setMaxSpoutPending(5);
if (flag == 0) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort));
- Thread.sleep(60 * 1000);
- cluster.killTopology("test_wordCounter_for_redis");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort));) {
+ Thread.sleep(60 * 1000);
+ }
System.exit(0);
} else if(flag == 1) {
conf.setNumWorkers(3);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java
----------------------------------------------------------------------
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java
index 384f97c..b0cddcd 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java
@@ -19,6 +19,7 @@ package org.apache.storm.redis.trident;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.tuple.Fields;
@@ -77,11 +78,10 @@ public class WordCountTridentRedisMap {
Config conf = new Config();
conf.setMaxSpoutPending(5);
if (flag == 0) {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, redisPort));
- Thread.sleep(60 * 1000);
- cluster.killTopology("test_wordCounter_for_redis");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, redisPort));) {
+ Thread.sleep(60 * 1000);
+ }
System.exit(0);
} else if(flag == 1) {
conf.setNumWorkers(3);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-solr-examples/src/main/java/org/apache/storm/solr/topology/SolrTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-solr-examples/src/main/java/org/apache/storm/solr/topology/SolrTopology.java b/examples/storm-solr-examples/src/main/java/org/apache/storm/solr/topology/SolrTopology.java
index 92d90a3..3b5b1b6 100644
--- a/examples/storm-solr-examples/src/main/java/org/apache/storm/solr/topology/SolrTopology.java
+++ b/examples/storm-solr-examples/src/main/java/org/apache/storm/solr/topology/SolrTopology.java
@@ -22,6 +22,7 @@ import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.solr.config.SolrCommitStrategy;
@@ -49,13 +50,12 @@ public abstract class SolrTopology {
StormSubmitter.submitTopology(arg, config, topology);
}
- protected void submitTopologyLocalCluster(StormTopology topology, Config config) throws InterruptedException {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", config, topology);
- Thread.sleep(10000);
- System.out.println("Killing topology per client's request");
- cluster.killTopology("test");
- cluster.shutdown();
+ protected void submitTopologyLocalCluster(StormTopology topology, Config config) throws Exception {
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", config, topology);) {
+ Thread.sleep(10000);
+ System.out.println("Killing topology per client's request");
+ }
System.exit(0);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/clj/org/apache/storm/starter/clj/exclamation.clj
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/clj/org/apache/storm/starter/clj/exclamation.clj b/examples/storm-starter/src/clj/org/apache/storm/starter/clj/exclamation.clj
index 0f51951..eed1a91 100644
--- a/examples/storm-starter/src/clj/org/apache/storm/starter/clj/exclamation.clj
+++ b/examples/storm-starter/src/clj/org/apache/storm/starter/clj/exclamation.clj
@@ -32,11 +32,9 @@
"exclaim2" (bolt-spec {"exclaim1" :shuffle} exclamation-bolt :p 2)}))
(defn run-local! []
- (let [cluster (LocalCluster.)]
- (.submitTopology cluster "exclamation" {TOPOLOGY-DEBUG true} (mk-topology))
- (Utils/sleep 10000)
- (.killTopology cluster "exclamation")
- (.shutdown cluster)))
+ (with-open [cluster (LocalCluster.)
+ topo (.submitTopology cluster "exclamation" {TOPOLOGY-DEBUG true} (mk-topology))]
+ (Utils/sleep 10000)))
(defn submit-topology! [name]
(StormSubmitter/submitTopologyWithProgressBar
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/clj/org/apache/storm/starter/clj/rolling_top_words.clj
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/clj/org/apache/storm/starter/clj/rolling_top_words.clj b/examples/storm-starter/src/clj/org/apache/storm/starter/clj/rolling_top_words.clj
index 53ba0a8..937ef70 100644
--- a/examples/storm-starter/src/clj/org/apache/storm/starter/clj/rolling_top_words.clj
+++ b/examples/storm-starter/src/clj/org/apache/storm/starter/clj/rolling_top_words.clj
@@ -39,10 +39,9 @@
(total-rankings-bolt 5 2))})))
(defn run-local! []
- (let [cluster (LocalCluster.)]
- (.submitTopology cluster "slidingWindowCounts" {TOPOLOGY-DEBUG true} (mk-topology))
- (Utils/sleep 60000)
- (.shutdown cluster)))
+ (with-open [cluster (LocalCluster.)
+ topo (.submitTopology cluster "slidingWindowCounts" {TOPOLOGY-DEBUG true} (mk-topology))]
+ (Utils/sleep 60000)))
(defn submit-topology! [name]
(StormSubmitter/submitTopology
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/clj/org/apache/storm/starter/clj/word_count.clj
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/clj/org/apache/storm/starter/clj/word_count.clj b/examples/storm-starter/src/clj/org/apache/storm/starter/clj/word_count.clj
index c35cc1f..cefa1e5 100644
--- a/examples/storm-starter/src/clj/org/apache/storm/starter/clj/word_count.clj
+++ b/examples/storm-starter/src/clj/org/apache/storm/starter/clj/word_count.clj
@@ -75,11 +75,9 @@
:p 6)}))
(defn run-local! []
- (let [cluster (LocalCluster.)]
- (.submitTopology cluster "word-count" {TOPOLOGY-DEBUG true} (mk-topology))
- (Thread/sleep 10000)
- (.shutdown cluster)
- ))
+ (with-open [cluster (LocalCluster.)
+ topo (.submitTopology cluster "word-count" {TOPOLOGY-DEBUG true} (mk-topology))]
+ (Thread/sleep 10000)))
(defn submit-topology! [name]
(StormSubmitter/submitTopology
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
index 2187da4..0f5f115 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
@@ -57,18 +57,17 @@ public class BasicDRPCTopology {
Config conf = new Config();
if (args == null || args.length == 0) {
- LocalDRPC drpc = new LocalDRPC();
- LocalCluster cluster = new LocalCluster();
+ try (LocalDRPC drpc = new LocalDRPC();
+ LocalCluster cluster = new LocalCluster()) {
- cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));
+ cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));
- for (String word : new String[]{ "hello", "goodbye" }) {
- System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
- }
+ for (String word : new String[]{ "hello", "goodbye" }) {
+ System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
+ }
- Thread.sleep(10000);
- drpc.shutdown();
- cluster.shutdown();
+ Thread.sleep(10000);
+ }
}
else {
conf.setNumWorkers(3);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
index 26e0430..7de8645 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -77,11 +78,10 @@ public class ExclamationTopology {
}
else {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", conf, builder.createTopology());
- Utils.sleep(10000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology());) {
+ Utils.sleep(10000);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
index f1e052e..5375ea6 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.LocalDRPC;
import org.apache.storm.drpc.DRPCSpout;
import org.apache.storm.drpc.ReturnResults;
@@ -47,19 +48,21 @@ public class ManualDRPC {
}
- public static void main(String[] args) throws Exception{
+ public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
- LocalDRPC drpc = new LocalDRPC();
+ try(LocalDRPC drpc = new LocalDRPC();
+ LocalCluster cluster = new LocalCluster();) {
- DRPCSpout spout = new DRPCSpout("exclamation", drpc);
- builder.setSpout("drpc", spout);
- builder.setBolt("exclaim", new ExclamationBolt(), 3).shuffleGrouping("drpc");
- builder.setBolt("return", new ReturnResults(), 3).shuffleGrouping("exclaim");
+ DRPCSpout spout = new DRPCSpout("exclamation", drpc);
+ builder.setSpout("drpc", spout);
+ builder.setBolt("exclaim", new ExclamationBolt(), 3).shuffleGrouping("drpc");
+ builder.setBolt("return", new ReturnResults(), 3).shuffleGrouping("exclaim");
- LocalCluster cluster = new LocalCluster();
- Config conf = new Config();
- cluster.submitTopology("exclaim", conf, builder.createTopology());
- System.out.println(drpc.execute("exclamation", "aaa"));
- System.out.println(drpc.execute("exclamation", "bbb"));
+ Config conf = new Config();
+ try (LocalTopology topo = cluster.submitTopology("exclaim", conf, builder.createTopology())) {
+ System.out.println(drpc.execute("exclamation", "aaa"));
+ System.out.println(drpc.execute("exclamation", "bbb"));
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
index 99c3da1..17cabf5 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -95,11 +96,10 @@ public class MultipleLoggerTopology {
conf.setNumWorkers(2);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
} else {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", conf, builder.createTopology());
- Utils.sleep(10000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology());) {
+ Utils.sleep(10000);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/PrintSampleStream.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/PrintSampleStream.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/PrintSampleStream.java
index 466fca0..140983c 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/PrintSampleStream.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/PrintSampleStream.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.utils.Utils;
@@ -29,7 +30,7 @@ import org.apache.storm.starter.bolt.PrinterBolt;
import org.apache.storm.starter.spout.TwitterSampleSpout;
public class PrintSampleStream {
- public static void main(String[] args) {
+ public static void main(String[] args) throws Exception {
String consumerKey = args[0];
String consumerSecret = args[1];
String accessToken = args[2];
@@ -44,15 +45,11 @@ public class PrintSampleStream {
builder.setBolt("print", new PrinterBolt())
.shuffleGrouping("twitter");
-
Config conf = new Config();
-
- LocalCluster cluster = new LocalCluster();
-
- cluster.submitTopology("test", conf, builder.createTopology());
-
- Utils.sleep(10000);
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology());) {
+ Utils.sleep(10000);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
index 13b2121..6b4aaa6 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
@@ -176,17 +176,15 @@ public class ReachTopology {
if (args == null || args.length == 0) {
conf.setMaxTaskParallelism(3);
- LocalDRPC drpc = new LocalDRPC();
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc));
+ try (LocalDRPC drpc = new LocalDRPC();
+ LocalCluster cluster = new LocalCluster();) {
+ cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc));
- String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
- for (String url : urlsToTry) {
- System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
+ String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
+ for (String url : urlsToTry) {
+ System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
+ }
}
-
- cluster.shutdown();
- drpc.shutdown();
}
else {
conf.setNumWorkers(6);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
index 19efbc5..788485d 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -96,11 +97,10 @@ public class ResourceAwareExampleTopology {
}
else {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", conf, builder.createTopology());
- Utils.sleep(10000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology());) {
+ Utils.sleep(10000);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
index b153372..0fd0d39 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.testing.FeederSpout;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
@@ -27,7 +28,7 @@ import org.apache.storm.utils.Utils;
import org.apache.storm.starter.bolt.SingleJoinBolt;
public class SingleJoinExample {
- public static void main(String[] args) {
+ public static void main(String[] args) throws Exception {
FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));
@@ -40,25 +41,25 @@ public class SingleJoinExample {
Config conf = new Config();
conf.setDebug(true);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("join-example", conf, builder.createTopology());
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("join-example", conf, builder.createTopology());) {
- for (int i = 0; i < 10; i++) {
- String gender;
- if (i % 2 == 0) {
- gender = "male";
+ for (int i = 0; i < 10; i++) {
+ String gender;
+ if (i % 2 == 0) {
+ gender = "male";
+ }
+ else {
+ gender = "female";
+ }
+ genderSpout.feed(new Values(i, gender));
}
- else {
- gender = "female";
+
+ for (int i = 9; i >= 0; i--) {
+ ageSpout.feed(new Values(i, i + 20));
}
- genderSpout.feed(new Values(i, gender));
- }
- for (int i = 9; i >= 0; i--) {
- ageSpout.feed(new Values(i, i + 20));
+ Utils.sleep(2000);
}
-
- Utils.sleep(2000);
- cluster.shutdown();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
index 90744f2..3993b95 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.topology.base.BaseWindowedBolt;
@@ -52,11 +53,10 @@ public class SlidingTupleTsTopology {
conf.setNumWorkers(1);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
} else {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", conf, builder.createTopology());
- Utils.sleep(40000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology());) {
+ Utils.sleep(40000);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
index 8477ad1..3c68f16 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -98,11 +99,10 @@ public class SlidingWindowTopology {
conf.setNumWorkers(1);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
} else {
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("test", conf, builder.createTopology());
- Utils.sleep(40000);
- cluster.killTopology("test");
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology());) {
+ Utils.sleep(40000);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
index 8ecfb3a..9e709a1 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
@@ -71,7 +71,7 @@ public class ThroughputVsLatency {
LocalCluster _local = null;
Nimbus.Client _client = null;
- public C(Map conf) {
+ public C(Map conf) throws Exception {
Map clusterConf = Utils.readStormConfig();
if (conf != null) {
clusterConf.putAll(conf);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java
index 312f83e..5a310f6 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.coordination.BatchOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.testing.MemoryTransactionalSpout;
@@ -160,15 +161,13 @@ public class TransactionalGlobalCount {
builder.setBolt("partial-count", new BatchCount(), 5).noneGrouping("spout");
builder.setBolt("sum", new UpdateGlobalCount()).globalGrouping("partial-count");
- LocalCluster cluster = new LocalCluster();
-
Config config = new Config();
config.setDebug(true);
config.setMaxSpoutPending(3);
-
- cluster.submitTopology("global-count-topology", config, builder.buildTopology());
-
- Thread.sleep(3000);
- cluster.shutdown();
+
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("global-count-topology", config, builder.buildTopology());) {
+ Thread.sleep(3000);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java
index 64689b0..4965565 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.coordination.BatchOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.testing.MemoryTransactionalSpout;
@@ -230,17 +231,13 @@ public class TransactionalWords {
builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
builder.setBolt("bucketize", new Bucketize()).noneGrouping("count");
builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));
-
-
- LocalCluster cluster = new LocalCluster();
-
Config config = new Config();
config.setDebug(true);
config.setMaxSpoutPending(3);
-
- cluster.submitTopology("top-n-topology", config, builder.buildTopology());
-
- Thread.sleep(3000);
- cluster.shutdown();
+
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("top-n-topology", config, builder.buildTopology());) {
+ Thread.sleep(3000);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
index e4a5711..c244fa1 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
@@ -19,6 +19,7 @@ package org.apache.storm.starter;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
+import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.StormSubmitter;
import org.apache.storm.task.ShellBolt;
import org.apache.storm.topology.BasicOutputCollector;
@@ -95,12 +96,10 @@ public class WordCountTopology {
else {
conf.setMaxTaskParallelism(3);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("word-count", conf, builder.createTopology());
-
- Thread.sleep(10000);
-
- cluster.shutdown();
+ try (LocalCluster cluster = new LocalCluster();
+ LocalTopology topo = cluster.submitTopology("word-count", conf, builder.createTopology());) {
+ Thread.sleep(10000);
+ }
}
}
}
[05/10] storm git commit: STORM-1281: LocalCluster,
testing4j and testing.clj to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/LocalCluster.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/LocalCluster.java b/storm-core/src/jvm/org/apache/storm/LocalCluster.java
new file mode 100644
index 0000000..3b18ac9
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/LocalCluster.java
@@ -0,0 +1,824 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.UnaryOperator;
+
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.cluster.ClusterStateContext;
+import org.apache.storm.cluster.ClusterUtils;
+import org.apache.storm.cluster.IStateStorage;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.daemon.Acker;
+import org.apache.storm.daemon.DaemonCommon;
+import org.apache.storm.daemon.Shutdownable;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.daemon.nimbus.Nimbus;
+import org.apache.storm.daemon.nimbus.Nimbus.StandaloneINimbus;
+import org.apache.storm.daemon.supervisor.ReadClusterState;
+import org.apache.storm.daemon.supervisor.StandaloneSupervisor;
+import org.apache.storm.daemon.supervisor.Supervisor;
+import org.apache.storm.executor.LocalExecutor;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.Credentials;
+import org.apache.storm.generated.KillOptions;
+import org.apache.storm.generated.Nimbus.Processor;
+import org.apache.storm.generated.NimbusSummary;
+import org.apache.storm.generated.RebalanceOptions;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.SubmitOptions;
+import org.apache.storm.generated.TopologyInfo;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.messaging.local.Context;
+import org.apache.storm.nimbus.ILeaderElector;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.security.auth.IGroupMappingServiceProvider;
+import org.apache.storm.security.auth.ThriftConnectionType;
+import org.apache.storm.security.auth.ThriftServer;
+import org.apache.storm.task.IBolt;
+import org.apache.storm.testing.InProcessZookeeper;
+import org.apache.storm.testing.NonRichBoltTracker;
+import org.apache.storm.testing.TmpPath;
+import org.apache.storm.testing.TrackedTopology;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.RegisteredGlobalState;
+import org.apache.storm.utils.StormCommonInstaller;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
+import org.apache.storm.utils.Utils;
+import org.apache.thrift.TException;
+import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A stand alone storm cluster that runs inside a single process.
+ * It is intended to be used for testing. Both internal testing for
+ * Apache Storm itself and for people building storm topologies.
+ *
+ * LocalCluster is an AutoCloseable so if you are using it in tests you can use
+ * a try block to be sure it is shut down.
+ *
+ * try (LocalCluster cluster = new LocalCluster()) {
+ * // Do some tests
+ * }
+ * // The cluster has been shut down.
+ */
+public class LocalCluster implements ILocalCluster {
+ private static final Logger LOG = LoggerFactory.getLogger(LocalCluster.class);
+
+ private static ThriftServer startNimbusDaemon(Map<String, Object> conf, Nimbus nimbus) {
+ ThriftServer ret = new ThriftServer(conf, new Processor<>(nimbus), ThriftConnectionType.NIMBUS);
+ LOG.info("Starting Nimbus server...");
+ new Thread(() -> ret.serve()).start();
+ return ret;
+ }
+
+ /**
+ * Simple way to configure a LocalCluster to meet your needs.
+ */
+ public static class Builder {
+ private int supervisors = 2;
+ private int portsPerSupervisor = 3;
+ private Map<String, Object> daemonConf = new HashMap<>();
+ private INimbus inimbus = null;
+ private IGroupMappingServiceProvider groupMapper = null;
+ private int supervisorSlotPortMin = 1024;
+ private boolean nimbusDaemon = false;
+ private UnaryOperator<Nimbus> nimbusWrapper = null;
+ private BlobStore store = null;
+ private IStormClusterState clusterState = null;
+ private ILeaderElector leaderElector = null;
+ private String trackId = null;
+ private boolean simulateTime = false;
+
+ /**
+ * Set the number of supervisors the cluster should have.
+ */
+ public Builder withSupervisors(int supervisors) {
+ if (supervisors < 0) {
+ throw new IllegalArgumentException("supervisors cannot be negative");
+ }
+ this.supervisors = supervisors;
+ return this;
+ }
+
+ /**
+ * Set the number of slots/ports each supervisor should have
+ */
+ public Builder withPortsPerSupervisor(int portsPerSupervisor) {
+ if (portsPerSupervisor < 0) {
+ throw new IllegalArgumentException("supervisor ports cannot be negative");
+ }
+ this.portsPerSupervisor = portsPerSupervisor;
+ return this;
+ }
+
+ /**
+ * Set the base config that the daemons should use.
+ */
+ public Builder withDaemonConf(Map<String, Object> conf) {
+ if (conf != null) {
+ this.daemonConf = new HashMap<>(conf);
+ }
+ return this;
+ }
+
+ /**
+ * Add an single key/value config to the daemon conf
+ */
+ public Builder withDaemonConf(String key, Object value) {
+ this.daemonConf.put(key, value);
+ return this;
+ }
+
+ /**
+ * Override the INimbus instance that nimbus will use.
+ */
+ public Builder withINimbus(INimbus inimbus) {
+ this.inimbus = inimbus;
+ return this;
+ }
+
+ /**
+ * Override the code that maps users to groups for authorization.
+ */
+ public Builder withGroupMapper(IGroupMappingServiceProvider groupMapper) {
+ this.groupMapper = groupMapper;
+ return this;
+ }
+
+ /**
+ * When assigning ports to worker slots start at minPort.
+ */
+ public Builder withSupervisorSlotPortMin(Number minPort) {
+ int port = 1024;
+ if (minPort == null) {
+ LOG.warn("Number is null... {}", minPort);
+ } else {
+ port = minPort.intValue();
+ }
+ if (port <= 0) {
+ throw new IllegalArgumentException("port must be positive");
+ }
+ this.supervisorSlotPortMin = port;
+ return this;
+ }
+
+ /**
+ * Have the local nimbus actually launch a thrift server. This is intended to
+ * be used mostly for internal storm testing.
+ */
+ public Builder withNimbusDaemon() {
+ return withNimbusDaemon(true);
+ }
+
+ /**
+ * If nimbusDaemon is true the local nimbus will launch a thrift server. This is intended to
+ * be used mostly for internal storm testing.
+ */
+ public Builder withNimbusDaemon(Boolean nimbusDaemon) {
+ if (nimbusDaemon == null) {
+ nimbusDaemon = false;
+ LOG.warn("nimbusDaemon is null");
+ }
+ this.nimbusDaemon = nimbusDaemon;
+ return this;
+ }
+
+ /**
+ * Turn on simulated time in the cluster. This allows someone to simulate long periods of
+ * time for timeouts etc when testing nimbus/supervisors themselves. NOTE: that this only
+ * works for code that uses the {@link org.apache.storm.utils.Time} class for time management
+ * so it will not work in all cases.
+ */
+ public Builder withSimulatedTime() {
+ return withSimulatedTime(true);
+ }
+
+ /**
+ * Turn on simulated time in the cluster. This allows someone to simulate long periods of
+ * time for timeouts etc when testing nimbus/supervisors themselves. NOTE: that this only
+ * works for code that uses the {@link org.apache.storm.utils.Time} class for time management
+ * so it will not work in all cases.
+ */
+ public Builder withSimulatedTime(boolean simulateTime) {
+ this.simulateTime = simulateTime;
+ return this;
+ }
+
+ /**
+ * Before nimbus is created/used call nimbusWrapper on it first and use the
+ * result instead. This is intended for internal testing only, and it here to
+ * allow a mocking framework to spy on the nimbus class.
+ */
+ public Builder withNimbusWrapper(UnaryOperator<Nimbus> nimbusWrapper) {
+ this.nimbusWrapper = nimbusWrapper;
+ return this;
+ }
+
+ /**
+ * Use the following blobstore instead of the one in the config.
+ * This is intended mostly for internal testing with Mocks.
+ */
+ public Builder withBlobStore(BlobStore store) {
+ this.store = store;
+ return this;
+ }
+
+ /**
+ * Use the following clusterState instead of the one in the config.
+ * This is intended mostly for internal testing with Mocks.
+ */
+ public Builder withClusterState(IStormClusterState clusterState) {
+ this.clusterState = clusterState;
+ return this;
+ }
+
+ /**
+ * Use the following leaderElector instead of the one in the config.
+ * This is intended mostly for internal testing with Mocks.
+ */
+ public Builder withLeaderElector(ILeaderElector leaderElector) {
+ this.leaderElector = leaderElector;
+ return this;
+ }
+
+ /**
+ * A tracked cluster can run tracked topologies.
+ * See {@link org.apache.storm.testing.TrackedTopology} for more information
+ * on tracked topologies.
+ * @param trackId an arbitrary unique id that is used to keep track of tracked topologies
+ */
+ public Builder withTracked(String trackId) {
+ this.trackId = trackId;
+ return this;
+ }
+
+ /**
+ * A tracked cluster can run tracked topologies.
+ * See {@link org.apache.storm.testing.TrackedTopology} for more information
+ * on tracked topologies.
+ */
+ public Builder withTracked() {
+ this.trackId = Utils.uuid();
+ return this;
+ }
+
+ /**
+ * @return the LocalCluster
+ * @throws Exception on any one of many different errors.
+ * This is intended for testing so yes it is ugly and throws Exception...
+ */
+ public LocalCluster build() throws Exception {
+ return new LocalCluster(this);
+ }
+ }
+
+ private static class TrackedStormCommon extends StormCommon {
+ private final String id;
+ public TrackedStormCommon(String id) {
+ this.id = id;
+ }
+
+ @Override
+ public IBolt makeAckerBoltImpl() {
+ return new NonRichBoltTracker(new Acker(), id);
+ }
+ }
+
+ private final Nimbus nimbus;
+ //This is very private and does not need to be exposed
+ private final AtomicInteger portCounter;
+ private final Map<String, Object> daemonConf;
+ private final List<Supervisor> supervisors;
+ private final IStateStorage state;
+ private final IStormClusterState clusterState;
+ private final List<TmpPath> tmpDirs;
+ private final InProcessZookeeper zookeeper;
+ private final IContext sharedContext;
+ private final ThriftServer thriftServer;
+ private final String trackId;
+ private final StormCommonInstaller commonInstaller;
+ private final SimulatedTime time;
+
+ /**
+ * Create a default LocalCluster
+ * @throws Exception on any error
+ */
+ public LocalCluster() throws Exception {
+ this(new Builder().withDaemonConf(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true));
+ }
+
+ /**
+ * Create a LocalCluster that connects to an existing Zookeeper instance
+ * @param zkHost the host for ZK
+ * @param zkPort the port for ZK
+ * @throws Exception on any error
+ */
+ public LocalCluster(String zkHost, Long zkPort) throws Exception {
+ this(new Builder().withDaemonConf(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)
+ .withDaemonConf(Config.STORM_ZOOKEEPER_SERVERS, Arrays.asList(zkHost))
+ .withDaemonConf(Config.STORM_ZOOKEEPER_PORT, zkPort));
+ }
+
+ @SuppressWarnings("deprecation")
+ private LocalCluster(Builder builder) throws Exception {
+ if (builder.simulateTime) {
+ time = new SimulatedTime();
+ } else {
+ time = null;
+ }
+ this.trackId = builder.trackId;
+ if (trackId != null) {
+ ConcurrentHashMap<String, AtomicInteger> metrics = new ConcurrentHashMap<>();
+ metrics.put("spout-emitted", new AtomicInteger(0));
+ metrics.put("transferred", new AtomicInteger(0));
+ metrics.put("processed", new AtomicInteger(0));
+ this.commonInstaller = new StormCommonInstaller(new TrackedStormCommon(this.trackId));
+ LOG.warn("Adding tracked metrics for ID {}", this.trackId);
+ RegisteredGlobalState.setState(this.trackId, metrics);
+ LocalExecutor.setTrackId(this.trackId);
+ } else {
+ this.commonInstaller = null;
+ }
+
+ this.tmpDirs = new ArrayList<>();
+ this.supervisors = new ArrayList<>();
+ TmpPath nimbusTmp = new TmpPath();
+ this.tmpDirs.add(nimbusTmp);
+ Map<String, Object> conf = ConfigUtils.readStormConfig();
+ conf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
+ conf.put(Config.ZMQ_LINGER_MILLIS, 0);
+ conf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, false);
+ conf.put(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS, 50);
+ conf.put(Config.STORM_CLUSTER_MODE, "local");
+ conf.put(Config.BLOBSTORE_SUPERUSER, System.getProperty("user.name"));
+ conf.put(Config.BLOBSTORE_DIR, nimbusTmp.getPath());
+
+ InProcessZookeeper zookeeper = null;
+ if (!builder.daemonConf.containsKey(Config.STORM_ZOOKEEPER_SERVERS)) {
+ zookeeper = new InProcessZookeeper();
+ conf.put(Config.STORM_ZOOKEEPER_PORT, zookeeper.getPort());
+ conf.put(Config.STORM_ZOOKEEPER_SERVERS, Arrays.asList("localhost"));
+ }
+ this.zookeeper = zookeeper;
+ conf.putAll(builder.daemonConf);
+ this.daemonConf = new HashMap<>(conf);
+
+ this.portCounter = new AtomicInteger(builder.supervisorSlotPortMin);
+ ClusterStateContext cs = new ClusterStateContext();
+ this.state = ClusterUtils.mkStateStorage(this.daemonConf, null, null, cs);
+ if (builder.clusterState == null) {
+ clusterState = ClusterUtils.mkStormClusterState(this.daemonConf, null, cs);
+ } else {
+ this.clusterState = builder.clusterState;
+ }
+ //Set it for nimbus only
+ conf.put(Config.STORM_LOCAL_DIR, nimbusTmp.getPath());
+ Nimbus nimbus = new Nimbus(conf, builder.inimbus == null ? new StandaloneINimbus() : builder.inimbus,
+ this.getClusterState(), null, builder.store, builder.leaderElector, builder.groupMapper);
+ if (builder.nimbusWrapper != null) {
+ nimbus = builder.nimbusWrapper.apply(nimbus);
+ }
+ this.nimbus = nimbus;
+ this.nimbus.launchServer();
+ IContext context = null;
+ if (!Utils.getBoolean(this.daemonConf.get(Config.STORM_LOCAL_MODE_ZMQ), false)) {
+ context = new Context();
+ context.prepare(this.daemonConf);
+ }
+ this.sharedContext = context;
+ this.thriftServer = builder.nimbusDaemon ? startNimbusDaemon(this.daemonConf, this.nimbus) : null;
+
+ for (int i = 0; i < builder.supervisors; i++) {
+ addSupervisor(builder.portsPerSupervisor, null, null);
+ }
+
+ //Wait for a leader to be elected (or topology submission can be rejected)
+ try {
+ long timeoutAfter = System.currentTimeMillis() + 10_000;
+ while (!hasLeader()) {
+ if (timeoutAfter > System.currentTimeMillis()) {
+ throw new IllegalStateException("Timed out waiting for nimbus to become the leader");
+ }
+ Thread.sleep(1);
+ }
+ } catch (Exception e) {
+ //Ignore any exceptions we might be doing a test for authentication
+ }
+ }
+
+ private boolean hasLeader() throws AuthorizationException, TException {
+ ClusterSummary summary = getNimbus().getClusterInfo();
+ if (summary.is_set_nimbuses()) {
+ for (NimbusSummary sum: summary.get_nimbuses()) {
+ if (sum.is_isLeader()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * @return Nimbus itself so you can interact with it directly, if needed.
+ */
+ public Nimbus getNimbus() {
+ return nimbus;
+ }
+
+ /**
+ * @return the base config for the daemons.
+ */
+ public Map<String, Object> getDaemonConf() {
+ return new HashMap<>(daemonConf);
+ }
+
+ public static final KillOptions KILL_NOW = new KillOptions();
+ static {
+ KILL_NOW.set_wait_secs(0);
+ }
+
+ /**
+ * When running a topology locally, for tests etc. It is helpful to be sure
+ * that the topology is dead before the test exits. This is an AutoCloseable
+ * topology that not only gives you access to the compiled StormTopology
+ * but also will kill the topology when it closes.
+ *
+ * try (LocalTopology testTopo = cluster.submitTopology("testing", ...)) {
+ * // Run Some test
+ * }
+ * // The topology has been killed
+ */
+ public class LocalTopology extends StormTopology implements ILocalTopology {
+ private static final long serialVersionUID = 6145919776650637748L;
+ private final String topoName;
+
+ public LocalTopology(String topoName, StormTopology topo) {
+ super(topo);
+ this.topoName = topoName;
+ }
+
+ @Override
+ public void close() throws TException {
+ killTopologyWithOpts(topoName, KILL_NOW);
+ }
+ }
+
+ @Override
+ public LocalTopology submitTopology(String topologyName, Map<String, Object> conf, StormTopology topology)
+ throws TException {
+ if (!Utils.isValidConf(conf)) {
+ throw new IllegalArgumentException("Topology conf is not json-serializable");
+ }
+ getNimbus().submitTopology(topologyName, null, JSONValue.toJSONString(conf), topology);
+
+ ISubmitterHook hook = (ISubmitterHook) Utils.getConfiguredClass(conf, Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN);
+ if (hook != null) {
+ TopologyInfo topologyInfo = Utils.getTopologyInfo(topologyName, null, conf);
+ try {
+ hook.notify(topologyInfo, conf, topology);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return new LocalTopology(topologyName, topology);
+ }
+
+ @Override
+ public LocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, StormTopology topology, SubmitOptions submitOpts)
+ throws TException {
+ if (!Utils.isValidConf(conf)) {
+ throw new IllegalArgumentException("Topology conf is not json-serializable");
+ }
+ getNimbus().submitTopologyWithOpts(topologyName, null, JSONValue.toJSONString(conf), topology, submitOpts);
+ return new LocalTopology(topologyName, topology);
+ }
+
+ @Override
+ public LocalTopology submitTopology(String topologyName, Map<String, Object> conf, TrackedTopology topology)
+ throws TException {
+ submitTopology(topologyName, conf, topology.getTopology());
+ return new LocalTopology(topologyName, topology.getTopology());
+ }
+
+ @Override
+ public LocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, TrackedTopology topology, SubmitOptions submitOpts)
+ throws TException {
+ submitTopologyWithOpts(topologyName, conf, topology.getTopology(), submitOpts);
+ return new LocalTopology(topologyName, topology.getTopology());
+ }
+
+ @Override
+ public void uploadNewCredentials(String topologyName, Credentials creds) throws TException {
+ getNimbus().uploadNewCredentials(topologyName, creds);
+ }
+
+ @Override
+ public void killTopology(String topologyName) throws TException {
+ getNimbus().killTopology(topologyName);
+ }
+
+ @Override
+ public void killTopologyWithOpts(String name, KillOptions options) throws TException {
+ getNimbus().killTopologyWithOpts(name, options);
+ }
+
+
+ @Override
+ public void activate(String topologyName) throws TException {
+ getNimbus().activate(topologyName);
+ }
+
+
+ @Override
+ public void deactivate(String topologyName) throws TException {
+ getNimbus().deactivate(topologyName);
+ }
+
+
+ @Override
+ public void rebalance(String name, RebalanceOptions options) throws TException {
+ getNimbus().rebalance(name, options);
+ }
+
+ @Override
+ public void shutdown() {
+ try {
+ close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ @Override
+ public String getTopologyConf(String id) throws TException {
+ return getNimbus().getTopologyConf(id);
+ }
+
+
+ @Override
+ public StormTopology getTopology(String id) throws TException {
+ return getNimbus().getTopology(id);
+ }
+
+
+ @Override
+ public ClusterSummary getClusterInfo() throws TException {
+ return getNimbus().getClusterInfo();
+ }
+
+
+ @Override
+ public TopologyInfo getTopologyInfo(String id) throws TException {
+ return getNimbus().getTopologyInfo(id);
+ }
+
+ @Override
+ public synchronized void close() throws Exception {
+ nimbus.shutdown();
+ if (thriftServer != null) {
+ LOG.info("shutting down thrift server");
+ try {
+ thriftServer.stop();
+ } catch (Exception e) {
+ LOG.info("failed to stop thrift", e);
+ }
+ }
+ if (state != null) {
+ state.close();
+ }
+ if (getClusterState() != null) {
+ getClusterState().disconnect();
+ }
+ for (Supervisor s: supervisors) {
+ s.shutdownAllWorkers(null, ReadClusterState.THREAD_DUMP_ON_ERROR);
+ s.close();
+ }
+ ProcessSimulator.killAllProcesses();
+ if (zookeeper != null) {
+ LOG.info("Shutting down in process zookeeper");
+ zookeeper.close();
+ LOG.info("Done shutting down in process zookeeper");
+ }
+
+ for (TmpPath p: tmpDirs) {
+ p.close();
+ }
+
+ if (this.trackId != null) {
+ LOG.warn("Clearing tracked metrics for ID {}", this.trackId);
+ LocalExecutor.clearTrackId();
+ RegisteredGlobalState.clearState(this.trackId);
+ }
+
+ if (this.commonInstaller != null) {
+ this.commonInstaller.close();
+ }
+
+ if (time != null) {
+ time.close();
+ }
+ }
+
+ /**
+ * Get a specific Supervisor. This is intended mostly for internal testing.
+ * @param id the id of the supervisor
+ */
+ public synchronized Supervisor getSupervisor(String id) {
+ for (Supervisor s: supervisors) {
+ if (id.equals(s.getId())) {
+ return s;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Kill a specific supervisor. This is intended mostly for internal testing.
+ * @param id the id of the supervisor
+ */
+ public synchronized void killSupervisor(String id) {
+ for (Iterator<Supervisor> it = supervisors.iterator(); it.hasNext();) {
+ Supervisor s = it.next();
+ if (id.equals(s.getId())) {
+ it.remove();
+ s.close();
+ //tmpDir will be handled separately
+ return;
+ }
+ }
+ }
+
+ /**
+ * Add another supervisor to the topology. This is intended mostly for internal testing.
+ */
+ public Supervisor addSupervisor() throws Exception {
+ return addSupervisor(null, null, null);
+ }
+
+ /**
+ * Add another supervisor to the topology. This is intended mostly for internal testing.
+ * @param ports the number of ports/slots the supervisor should have
+ */
+ public Supervisor addSupervisor(Number ports) throws Exception {
+ return addSupervisor(ports, null, null);
+ }
+
+ /**
+ * Add another supervisor to the topology. This is intended mostly for internal testing.
+ * @param ports the number of ports/slots the supervisor should have
+ * @param id the id of the new supervisor, so you can find it later.
+ */
+ public Supervisor addSupervisor(Number ports, String id) throws Exception {
+ return addSupervisor(ports, null, id);
+ }
+
+ /**
+ * Add another supervisor to the topology. This is intended mostly for internal testing.
+ * @param ports the number of ports/slots the supervisor should have
+ * @param conf any config values that should be added/over written in the daemon conf of the cluster.
+ * @param id the id of the new supervisor, so you can find it later.
+ */
+ public synchronized Supervisor addSupervisor(Number ports, Map<String, Object> conf, String id) throws Exception {
+ if (ports == null) {
+ ports = 2;
+ }
+ TmpPath tmpDir = new TmpPath();
+ tmpDirs.add(tmpDir);
+
+ List<Integer> portNumbers = new ArrayList<>(ports.intValue());
+ for (int i = 0; i < ports.intValue(); i++) {
+ portNumbers.add(portCounter.getAndIncrement());
+ }
+
+ Map<String, Object> superConf = new HashMap<>(daemonConf);
+ if (conf != null) {
+ superConf.putAll(conf);
+ }
+ superConf.put(Config.STORM_LOCAL_DIR, tmpDir.getPath());
+ superConf.put(Config.SUPERVISOR_SLOTS_PORTS, portNumbers);
+
+ final String superId = id == null ? Utils.uuid() : id;
+ ISupervisor isuper = new StandaloneSupervisor() {
+ @Override
+ public String generateSupervisorId() {
+ return superId;
+ }
+ };
+ if (!ConfigUtils.isLocalMode(superConf)) {
+ throw new IllegalArgumentException("Cannot start server in distrubuted mode!");
+ }
+
+ Supervisor s = new Supervisor(superConf, sharedContext, isuper);
+ s.launch();
+ supervisors.add(s);
+ return s;
+ }
+
+ private boolean areAllSupervisorsWaiting() {
+ boolean ret = true;
+ for (Supervisor s: supervisors) {
+ ret = ret && s.isWaiting();
+ }
+ return ret;
+ }
+
+ private static boolean areAllWorkersWaiting() {
+ boolean ret = true;
+ for (Shutdownable s: ProcessSimulator.getAllProcessHandles()) {
+ if (s instanceof DaemonCommon) {
+ ret = ret && ((DaemonCommon)s).isWaiting();
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Wait for the cluster to be idle. This is intended to be used with
+ * Simulated time and is for internal testing.
+ * @throws InterruptedException if interrupted while waiting.
+ * @throws AssertionError if the cluster did not come to an idle point with
+ * a timeout.
+ */
+ public void waitForIdle() throws InterruptedException {
+ waitForIdle(Testing.TEST_TIMEOUT_MS);
+ }
+
+ /**
+ * Wait for the cluster to be idle. This is intended to be used with
+ * Simulated time and is for internal testing.
+ * @param timeoutMs the number of ms to wait before throwing an error.
+ * @throws InterruptedException if interrupted while waiting.
+ * @throws AssertionError if the cluster did not come to an idle point with
+ * a timeout.
+ */
+ public void waitForIdle(long timeoutMs) throws InterruptedException {
+ Random rand = ThreadLocalRandom.current();
+ //wait until all workers, supervisors, and nimbus is waiting
+ final long endTime = System.currentTimeMillis() + timeoutMs;
+ while (!(nimbus.isWaiting() &&
+ areAllSupervisorsWaiting() &&
+ areAllWorkersWaiting())) {
+ if (System.currentTimeMillis() >= endTime) {
+ LOG.info("Cluster was not idle in {} ms", timeoutMs);
+ LOG.info(Utils.threadDump());
+ throw new AssertionError("Test timed out (" + timeoutMs + "ms) cluster not idle");
+ }
+ Thread.sleep(rand.nextInt(20));
+ }
+ }
+
+ @Override
+ public void advanceClusterTime(int secs) throws InterruptedException {
+ advanceClusterTime(secs, 1);
+ }
+
+ @Override
+ public void advanceClusterTime(int secs, int incSecs) throws InterruptedException {
+ for (int amountLeft = secs; amountLeft > 0; amountLeft -= incSecs) {
+ int diff = Math.min(incSecs, amountLeft);
+ Time.advanceTimeSecs(diff);
+ waitForIdle();
+ }
+ }
+
+ @Override
+ public IStormClusterState getClusterState() {
+ return clusterState;
+ }
+
+ @Override
+ public String getTrackedId() {
+ return trackId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/LocalDRPC.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/LocalDRPC.java b/storm-core/src/jvm/org/apache/storm/LocalDRPC.java
index ccdf634..e3f34d1 100644
--- a/storm-core/src/jvm/org/apache/storm/LocalDRPC.java
+++ b/storm-core/src/jvm/org/apache/storm/LocalDRPC.java
@@ -17,24 +17,30 @@
*/
package org.apache.storm;
+import java.util.Map;
+
import org.apache.storm.daemon.DrpcServer;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.DRPCExecutionException;
import org.apache.storm.generated.DRPCRequest;
import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.ServiceRegistry;
-import org.apache.storm.utils.Utils;
import org.apache.thrift.TException;
-import java.util.Map;
-
+/**
+ * A Local way to test DRPC
+ *
+ * try (LocalDRPC drpc = new LocalDRPC()) {
+ * // Do tests
+ * }
+ */
public class LocalDRPC implements ILocalDRPC {
private final DrpcServer handler;
private final String serviceId;
public LocalDRPC() {
- Map conf = ConfigUtils.readStormConfig();
+ Map<String, Object> conf = ConfigUtils.readStormConfig();
handler = new DrpcServer(conf);
serviceId = ServiceRegistry.registerService(handler);
}
@@ -61,12 +67,17 @@ public class LocalDRPC implements ILocalDRPC {
@Override
public void shutdown() {
- ServiceRegistry.unregisterService(this.serviceId);
- this.handler.close();
+ close();
}
@Override
public DRPCRequest fetchRequest(String functionName) throws AuthorizationException, TException {
return handler.fetchRequest(functionName);
}
+
+ @Override
+ public void close() {
+ ServiceRegistry.unregisterService(this.serviceId);
+ this.handler.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/Testing.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/Testing.java b/storm-core/src/jvm/org/apache/storm/Testing.java
new file mode 100644
index 0000000..be90ae8
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/Testing.java
@@ -0,0 +1,716 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.generated.Bolt;
+import org.apache.storm.generated.GlobalStreamId;
+import org.apache.storm.generated.Grouping;
+import org.apache.storm.generated.KillOptions;
+import org.apache.storm.generated.SpoutSpec;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.StreamInfo;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.testing.CompletableSpout;
+import org.apache.storm.testing.CompleteTopologyParam;
+import org.apache.storm.testing.FixedTuple;
+import org.apache.storm.testing.FixedTupleSpout;
+import org.apache.storm.testing.MkClusterParam;
+import org.apache.storm.testing.MkTupleParam;
+import org.apache.storm.testing.MockedSources;
+import org.apache.storm.testing.TestJob;
+import org.apache.storm.testing.TrackedTopology;
+import org.apache.storm.testing.TupleCaptureBolt;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.TupleImpl;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.RegisteredGlobalState;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
+import org.apache.storm.utils.Utils;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A utility that helps with testing topologies, Bolts and Spouts.
+ */
+public class Testing {
+ private static final Logger LOG = LoggerFactory.getLogger(Testing.class);
+ /**
+ * The default amount of wall time should be spent waiting for
+ * specific conditions to happen. Default is 10 seconds unless
+ * the environment variable STORM_TEST_TIMEOUT_MS is set.
+ */
+ public static final int TEST_TIMEOUT_MS;
+ static {
+ int timeout = 10_000;
+ try {
+ timeout = Integer.parseInt(System.getenv("STORM_TEST_TIMEOUT_MS"));
+ } catch (Exception e) {
+ //Ignored, will go with default timeout
+ }
+ TEST_TIMEOUT_MS = timeout;
+ }
+
+ /**
+ * Simply produces a boolean to see if a specific state is true or false.
+ */
+ public static interface Condition {
+ public boolean exec();
+ }
+
+ /**
+ * Continue to execute body repeatedly until condition is true or TEST_TIMEOUT_MS has
+ * passed
+ * @param condition what we are waiting for
+ * @param body what to run in the loop
+ * @throws AssertionError if teh loop timed out.
+ */
+ public static void whileTimeout(Condition condition, Runnable body) {
+ whileTimeout(TEST_TIMEOUT_MS, condition, body);
+ }
+
+ /**
+ * Continue to execute body repeatedly until condition is true or TEST_TIMEOUT_MS has
+ * passed
+ * @param the number of ms to wait before timing out.
+ * @param condition what we are waiting for
+ * @param body what to run in the loop
+ * @throws AssertionError if teh loop timed out.
+ */
+ public static void whileTimeout(long timeoutMs, Condition condition, Runnable body) {
+ long endTime = System.currentTimeMillis() + timeoutMs;
+ LOG.debug("Looping until {}", condition);
+ while (condition.exec()) {
+ if (System.currentTimeMillis() > endTime) {
+ LOG.info("Condition {} not met in {} ms", condition, timeoutMs);
+ LOG.info(Utils.threadDump());
+ throw new AssertionError("Test timed out (" + timeoutMs + "ms) " + condition);
+ }
+ body.run();
+ }
+ LOG.debug("Condition met {}", condition);
+ }
+
+ /**
+ * Convenience method for data.stream.allMatch(pred)
+ */
+ public static <T> boolean isEvery(Collection<T> data, Predicate<T> pred) {
+ return data.stream().allMatch(pred);
+ }
+
+ /**
+ * Run with simulated time
+ * @deprecated use ```
+ * try (Time.SimulatedTime time = new Time.SimulatedTime()) {
+ * ...
+ * }
+ * ```
+ * @param code what to run
+ */
+ @Deprecated
+ public static void withSimulatedTime(Runnable code) {
+ try (SimulatedTime st = new SimulatedTime()) {
+ code.run();
+ }
+ }
+
+ private static LocalCluster cluster(MkClusterParam param, boolean simulated) throws Exception {
+ return cluster(param, null, simulated);
+ }
+
+ private static LocalCluster cluster(MkClusterParam param) throws Exception {
+ return cluster(param, null, false);
+ }
+
+ private static LocalCluster cluster(MkClusterParam param, String id, boolean simulated) throws Exception {
+ Integer supervisors = param.getSupervisors();
+ if (supervisors == null) {
+ supervisors = 2;
+ }
+ Integer ports = param.getPortsPerSupervisor();
+ if (ports == null) {
+ ports = 3;
+ }
+ Map<String, Object> conf = param.getDaemonConf();
+ if (conf == null) {
+ conf = new HashMap<>();
+ }
+ return new LocalCluster.Builder()
+ .withSupervisors(supervisors)
+ .withPortsPerSupervisor(ports)
+ .withDaemonConf(conf)
+ .withNimbusDaemon(param.isNimbusDaemon())
+ .withTracked(id)
+ .withSimulatedTime(simulated)
+ .build();
+ }
+
+ /**
+ * Run with a local cluster
+ * @deprecated use ```
+ * try (LocalCluster cluster = new LocalCluster()) {
+ * ...
+ * }
+ * ```
+ * @param code what to run
+ */
+ @Deprecated
+ public static void withLocalCluster(TestJob code) {
+ withLocalCluster(new MkClusterParam(), code);
+ }
+
+ /**
+ * Run with a local cluster
+ * @deprecated use ```
+ * try (LocalCluster cluster = new LocalCluster.Builder()....build()) {
+ * ...
+ * }
+ * ```
+ * @param param configs to set in the cluster
+ * @param code what to run
+ */
+ @Deprecated
+ public static void withLocalCluster(MkClusterParam param, TestJob code) {
+ try (LocalCluster lc = cluster(param)) {
+ code.run(lc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Run with a local cluster
+ * @deprecated use ```
+ * try (LocalCluster cluster = new LocalCluster.Builder()....build()) {
+ * ...
+ * }
+ * ```
+ * @param clusterConf some configs to set in the cluster
+ */
+ @Deprecated
+ public static ILocalCluster getLocalCluster(Map<String, Object> clusterConf) {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> conf = (Map<String, Object>) clusterConf.get("daemon-conf");
+ if (conf == null) {
+ conf = new HashMap<>();
+ }
+ Number supervisors = (Number) clusterConf.getOrDefault("supervisors", 2);
+ Number ports = (Number) clusterConf.getOrDefault("ports-per-supervisor", 3);
+ INimbus inimbus = (INimbus) clusterConf.get("inimbus");
+ Number portMin = (Number) clusterConf.getOrDefault("supervisor-slot-port-min", 1024);
+ Boolean nimbusDaemon = (Boolean) clusterConf.getOrDefault("nimbus-daemon", false);
+ try {
+ return new LocalCluster.Builder()
+ .withSupervisors(supervisors.intValue())
+ .withDaemonConf(conf)
+ .withPortsPerSupervisor(ports.intValue())
+ .withINimbus(inimbus)
+ .withSupervisorSlotPortMin(portMin)
+ .withNimbusDaemon(nimbusDaemon)
+ .build();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Run with a local cluster
+ * @deprecated use ```
+ * try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().build()) {
+ * ...
+ * }
+ * ```
+ * @param code what to run
+ */
+ @Deprecated
+ public static void withSimulatedTimeLocalCluster(TestJob code) {
+ withSimulatedTimeLocalCluster(new MkClusterParam(), code);
+ }
+
+ /**
+ * Run with a local cluster
+ * @deprecated use ```
+ * try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime()....build()) {
+ * ...
+ * }
+ * ```
+ * @param param configs to set in the cluster
+ * @param code what to run
+ */
+ @Deprecated
+ public static void withSimulatedTimeLocalCluster(MkClusterParam param, TestJob code) {
+ try (LocalCluster lc = cluster(param, true)) {
+ code.run(lc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Run with a local cluster
+ * @deprecated use ```
+ * try (LocalCluster cluster = new LocalCluster.Builder().withTracked().build()) {
+ * ...
+ * }
+ * ```
+ * @param code what to run
+ */
+ @Deprecated
+ public static void withTrackedCluster(TestJob code) {
+ withTrackedCluster(new MkClusterParam(), code);
+ }
+
+ /**
+ * In a tracked topology some metrics are tracked. This provides a way to get those metrics.
+ * This is intended mostly for internal testing.
+ * @param id the id of the tracked cluster
+ * @param key the name of the metric to get.
+ * @return the metric
+ */
+ @SuppressWarnings("unchecked")
+ @Deprecated
+ public static int globalAmt(String id, String key) {
+ LOG.warn("Reading tracked metrics for ID {}", id);
+ return ((ConcurrentHashMap<String, AtomicInteger>)RegisteredGlobalState.getState(id)).get(key).get();
+ }
+
+ /**
+ * Run with a local tracked cluster
+ * @deprecated use ```
+ * try (LocalCluster cluster = new LocalCluster.Builder().withTracked()....build()) {
+ * ...
+ * }
+ * ```
+ * @param param configs to set in the cluster
+ * @param code what to run
+ */
+ @Deprecated
+ public static void withTrackedCluster(MkClusterParam param, TestJob code) {
+ try (LocalCluster lc = cluster(param, Utils.uuid(), true)) {
+ code.run(lc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * A topology that has all messages captured and can be read later on.
+ * This is intended mostly for internal testing.
+ * @param <T> the topology (tracked or regular)
+ */
+ public static final class CapturedTopology<T> {
+ public final T topology;
+ /**
+ * a Bolt that will hold all of the captured data.
+ */
+ public final TupleCaptureBolt capturer;
+
+ public CapturedTopology(T topology, TupleCaptureBolt capturer) {
+ this.topology = topology;
+ this.capturer = capturer;
+ }
+ }
+
+ /**
+ * Track and capture a topology.
+ * This is intended mostly for internal testing.
+ */
+ public static CapturedTopology<TrackedTopology> trackAndCaptureTopology(ILocalCluster cluster, StormTopology topology) {
+ CapturedTopology<StormTopology> captured = captureTopology(topology);
+ return new CapturedTopology<>(new TrackedTopology(captured.topology, cluster), captured.capturer);
+ }
+
+ /**
+ * Rewrites a topology so that all the tuples flowing through it are captured
+ * @param topology the topology to rewrite
+ * @return the modified topology and a new Bolt that can retrieve the
+ * captured tuples.
+ */
+ public static CapturedTopology<StormTopology> captureTopology(StormTopology topology) {
+ topology = topology.deepCopy(); //Don't modify the original
+
+ TupleCaptureBolt capturer = new TupleCaptureBolt();
+ Map<GlobalStreamId, Grouping> captureBoltInputs = new HashMap<>();
+ for (Map.Entry<String, SpoutSpec> spoutEntry : topology.get_spouts().entrySet()) {
+ String id = spoutEntry.getKey();
+ for (Entry<String, StreamInfo> streamEntry : spoutEntry.getValue().get_common().get_streams().entrySet()) {
+ String stream = streamEntry.getKey();
+ StreamInfo info = streamEntry.getValue();
+ if (info.is_direct()) {
+ captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
+ } else {
+ captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
+ }
+ }
+ }
+
+ for (Entry<String, Bolt> boltEntry : topology.get_bolts().entrySet()) {
+ String id = boltEntry.getKey();
+ for (Entry<String, StreamInfo> streamEntry : boltEntry.getValue().get_common().get_streams().entrySet()) {
+ String stream = streamEntry.getKey();
+ StreamInfo info = streamEntry.getValue();
+ if (info.is_direct()) {
+ captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
+ } else {
+ captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
+ }
+ }
+ }
+ topology.put_to_bolts(Utils.uuid(), new Bolt(Thrift.serializeComponentObject(capturer),
+ Thrift.prepareComponentCommon(captureBoltInputs, new HashMap<>(), null)));
+ return new CapturedTopology<>(topology, capturer);
+ }
+
+ /**
+ * Run a topology to completion capturing all of the messages that are emitted. This only works when all of the spouts are
+ * instances of {@link org.apache.storm.testing.CompletableSpout}
+ * @param cluster the cluster to submit the topology to
+ * @param topology the topology itself
+ * @return a map of the component to the list of tuples it emitted.
+ * @throws InterruptedException
+ * @throws TException on any error from nimbus.
+ */
+ public static Map<String, List<FixedTuple>> completeTopology(ILocalCluster cluster, StormTopology topology) throws InterruptedException, TException {
+ return completeTopology(cluster, topology, new CompleteTopologyParam());
+ }
+
+ /**
+ * Run a topology to completion capturing all of the messages that are emitted. This only works when all of the spouts are
+ * instances of {@link org.apache.storm.testing.CompletableSpout} or are overwritten by MockedSources in param
+ * @param cluster the cluster to submit the topology to
+ * @param topology the topology itself
+ * @param param parameters to describe how to complete a topology.
+ * @return a map of the component to the list of tuples it emitted.
+ * @throws InterruptedException
+ * @throws TException on any error from nimbus.
+ */
+ public static Map<String, List<FixedTuple>> completeTopology(ILocalCluster cluster, StormTopology topology,
+ CompleteTopologyParam param) throws TException, InterruptedException {
+ Map<String, List<FixedTuple>> ret = null;
+ IStormClusterState state = cluster.getClusterState();
+ CapturedTopology<StormTopology> capTopo = captureTopology(topology);
+ topology = capTopo.topology;
+ String topoName = param.getTopologyName();
+ if (topoName == null) {
+ topoName = "topologytest-" + Utils.uuid();
+ }
+
+ Map<String, SpoutSpec> spouts = topology.get_spouts();
+ MockedSources ms = param.getMockedSources();
+ if (ms != null) {
+ for (Entry<String, List<FixedTuple>> mocked: ms.getData().entrySet()) {
+ FixedTupleSpout newSpout = new FixedTupleSpout(mocked.getValue());
+ spouts.get(mocked.getKey()).set_spout_object(Thrift.serializeComponentObject(newSpout));
+ }
+ }
+ List<Object> spoutObjects = spouts.values().stream().
+ map((spec) -> Thrift.deserializeComponentObject(spec.get_spout_object())).collect(Collectors.toList());
+
+ for (Object o: spoutObjects) {
+ if (!(o instanceof CompletableSpout)) {
+ throw new RuntimeException("Cannot complete topology unless every spout is a CompletableSpout (or mocked to be); failed by " + o);
+ }
+ }
+
+ for (Object spout: spoutObjects) {
+ ((CompletableSpout)spout).startup();
+ }
+
+ cluster.submitTopology(topoName, param.getStormConf(), topology);
+
+ if (Time.isSimulating()) {
+ cluster.advanceClusterTime(11);
+ }
+
+ String topoId = state.getTopoId(topoName).get();
+ //Give the topology time to come up without using it to wait for the spouts to complete
+ simulateWait(cluster);
+ Integer timeoutMs = param.getTimeoutMs();
+ if (timeoutMs == null) {
+ timeoutMs = TEST_TIMEOUT_MS;
+ }
+ whileTimeout(timeoutMs,
+ () -> !isEvery(spoutObjects, (o) -> ((CompletableSpout)o).isExhausted()),
+ () -> {
+ try {
+ simulateWait(cluster);
+ } catch (Exception e) {
+ throw new RuntimeException();
+ }
+ });
+
+ KillOptions killOpts = new KillOptions();
+ killOpts.set_wait_secs(0);
+ cluster.killTopologyWithOpts(topoName, killOpts);
+
+ whileTimeout(timeoutMs,
+ () -> state.assignmentInfo(topoId, null) != null,
+ () -> {
+ try {
+ simulateWait(cluster);
+ } catch (Exception e) {
+ throw new RuntimeException();
+ }
+ });
+
+ if (param.getCleanupState()) {
+ for (Object o : spoutObjects) {
+ ((CompletableSpout)o).clean();
+ }
+ ret = capTopo.capturer.getAndRemoveResults();
+ } else {
+ ret = capTopo.capturer.getAndClearResults();
+ }
+
+ return ret;
+ }
+
+ /**
+ * If using simulated time simulate waiting for 10 seconds. This is intended for internal testing only.
+ */
+ public static void simulateWait(ILocalCluster cluster) throws InterruptedException {
+ if (Time.isSimulating()) {
+ cluster.advanceClusterTime(10);
+ Thread.sleep(100);
+ }
+ }
+
+ /**
+ * Get all of the tuples from a given component on the default stream
+ * @param results the results of running a completed topology
+ * @param componentId the id of the component to look at
+ * @return a list of the tuple values.
+ */
+ public static List<List<Object>> readTuples(Map<String, List<FixedTuple>> results, String componentId) {
+ return readTuples(results, componentId, Utils.DEFAULT_STREAM_ID);
+ }
+
+ /**
+ * Get all of the tuples from a given component on a given stream
+ * @param results the results of running a completed topology
+ * @param componentId the id of the component to look at
+ * @param streamId the id of the stream to look for.
+ * @return a list of the tuple values.
+ */
+ public static List<List<Object>> readTuples(Map<String, List<FixedTuple>> results, String componentId, String streamId) {
+ List<List<Object>> ret = new ArrayList<>();
+ List<FixedTuple> streamResult = results.get(componentId);
+ if (streamResult != null) {
+ for (FixedTuple tuple: streamResult) {
+ if (streamId.equals(tuple.stream)) {
+ ret.add(tuple.values);
+ }
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Create a tracked topology.
+ * @deprecated use {@link org.apache.storm.testing.TrackedTopology} directly.
+ */
+ @Deprecated
+ public static TrackedTopology mkTrackedTopology(ILocalCluster cluster, StormTopology topology) {
+ return new TrackedTopology(topology, cluster);
+ }
+
+ /**
+ * Simulated time wait for a tracked topology. This is intended for internal testing
+ */
+ public static void trackedWait(CapturedTopology<TrackedTopology> topo) {
+ topo.topology.trackedWait();
+ }
+
+ /**
+ * Simulated time wait for a tracked topology. This is intended for internal testing
+ */
+ public static void trackedWait(CapturedTopology<TrackedTopology> topo, Integer amt) {
+ topo.topology.trackedWait(amt);
+ }
+
+ /**
+ * Simulated time wait for a tracked topology. This is intended for internal testing
+ */
+ public static void trackedWait(CapturedTopology<TrackedTopology> topo, Integer amt, Integer timeoutMs) {
+ topo.topology.trackedWait(amt, timeoutMs);
+ }
+
+ /**
+ * Simulated time wait for a tracked topology. This is intended for internal testing
+ */
+ public static void trackedWait(TrackedTopology topo) {
+ topo.trackedWait();
+ }
+
+ /**
+ * Simulated time wait for a tracked topology. This is intended for internal testing
+ */
+ public static void trackedWait(TrackedTopology topo, Integer amt) {
+ topo.trackedWait(amt);
+ }
+
+ /**
+ * Simulated time wait for a tracked topology. This is intended for internal testing
+ */
+ public static void trackedWait(TrackedTopology topo, Integer amt, Integer timeoutMs) {
+ topo.trackedWait(amt, timeoutMs);
+ }
+
+ /**
+ * Simulated time wait for a cluster. This is intended for internal testing
+ */
+ public static void advanceClusterTime(ILocalCluster cluster, Integer secs) throws InterruptedException {
+ advanceClusterTime(cluster, secs, 1);
+ }
+
+ /**
+ * Simulated time wait for a cluster. This is intended for internal testing
+ */
+ public static void advanceClusterTime(ILocalCluster cluster, Integer secs, Integer step) throws InterruptedException {
+ cluster.advanceClusterTime(secs, step);
+ }
+
+ /**
+ * Count how many times each element appears in the Collection
+ * @param c a collection of values
+ * @return a map of the unique values in c to the count of those values.
+ */
+ public static <T> Map<T, Integer> multiset(Collection<T> c) {
+ Map<T, Integer> ret = new HashMap<T, Integer>();
+ for (T t: c) {
+ Integer i = ret.get(t);
+ if (i == null) {
+ i = new Integer(0);
+ }
+ i += 1;
+ ret.put(t, i);
+ }
+ return ret;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static void printRec(Object o, String prefix) {
+ if (o instanceof Collection) {
+ LOG.info("{} {} ({}) [",prefix,o, o.getClass());
+ for (Object sub: (Collection)o) {
+ printRec(sub, prefix + " ");
+ }
+ LOG.info("{} ]",prefix);
+ } else if (o instanceof Map) {
+ Map<?,?> m = (Map<?,?>)o;
+ LOG.info("{} {} ({}) {",prefix,o, o.getClass());
+ for (Map.Entry<?, ?> entry: m.entrySet()) {
+ printRec(entry.getKey(), prefix + " ");
+ LOG.info("{} ->", prefix);
+ printRec(entry.getValue(), prefix + " ");
+ }
+ LOG.info("{} }",prefix);
+ } else {
+ LOG.info("{} {} ({})", prefix, o, o.getClass());
+ }
+ }
+
+ /**
+ * Check if two collections are equivalent ignoring the order of elements.
+ */
+ public static <T> boolean multiseteq(Collection<T> a, Collection<T> b) {
+ boolean ret = multiset(a).equals(multiset(b));
+ if (! ret) {
+ printRec(multiset(a), "MS-A:");
+ printRec(multiset(b), "MS-B:");
+ }
+ return ret;
+ }
+
+ /**
+ * Create a {@link org.apache.storm.tuple.Tuple} for use with testing
+ * @param values the values to appear in the tuple
+ */
+ public static Tuple testTuple(List<Object> values) {
+ return testTuple(values, new MkTupleParam());
+ }
+
+ /**
+ * Create a {@link org.apache.storm.tuple.Tuple} for use with testing
+ * @param values the values to appear in the tuple
+ * @param param parametrs describing more details about the tuple
+ */
+ public static Tuple testTuple(List<Object> values, MkTupleParam param) {
+ String stream = param.getStream();
+ if (stream == null) {
+ stream = Utils.DEFAULT_STREAM_ID;
+ }
+
+ String component = param.getComponent();
+ if (component == null) {
+ component = "component";
+ }
+
+ int task = 1;
+
+ List<String> fields = param.getFields();
+ if (fields == null) {
+ fields = new ArrayList<>(values.size());
+ for (int i = 1; i <= values.size(); i++) {
+ fields.add("field"+i);
+ }
+ }
+
+ Map<Integer, String> taskToComp = new HashMap<>();
+ taskToComp.put(task, component);
+ Map<String, Map<String, Fields>> compToStreamToFields = new HashMap<>();
+ Map<String, Fields> streamToFields = new HashMap<>();
+ streamToFields.put(stream, new Fields(fields));
+ compToStreamToFields.put(component, streamToFields);
+
+ TopologyContext context= new TopologyContext(null,
+ ConfigUtils.readStormConfig(),
+ taskToComp,
+ null,
+ compToStreamToFields,
+ "test-storm-id",
+ null,
+ null,
+ 1,
+ null,
+ null,
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<>(),
+ new AtomicBoolean(false));
+ return new TupleImpl(context, values, 1, stream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/Thrift.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/Thrift.java b/storm-core/src/jvm/org/apache/storm/Thrift.java
index cde822f..9ed8bb2 100644
--- a/storm-core/src/jvm/org/apache/storm/Thrift.java
+++ b/storm-core/src/jvm/org/apache/storm/Thrift.java
@@ -278,7 +278,7 @@ public class Thrift {
public static SpoutSpec prepareSerializedSpoutDetails(IRichSpout spout, Map<String, StreamInfo> outputs) {
return new SpoutSpec(ComponentObject.serialized_java
- (Utils.javaSerialize(spout)), prepareComponentCommon(new HashMap(), outputs, null, null));
+ (Utils.javaSerialize(spout)), prepareComponentCommon(new HashMap<>(), outputs, null, null));
}
public static Bolt prepareSerializedBoltDetails(Map<GlobalStreamId, Grouping> inputs, IBolt bolt, Map<String, StreamInfo> outputs,
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java b/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
index c0b78d3..c550cfe 100644
--- a/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/blobstore/BlobStoreUtils.java
@@ -30,6 +30,7 @@ import org.apache.storm.utils.Utils;
import org.apache.storm.utils.ZookeeperAuthInfo;
import org.apache.curator.framework.CuratorFramework;
import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +49,8 @@ public class BlobStoreUtils {
private static final String BLOB_DEPENDENCIES_PREFIX = "dep-";
private static final Logger LOG = LoggerFactory.getLogger(BlobStoreUtils.class);
- public static CuratorFramework createZKClient(Map conf) {
+ public static CuratorFramework createZKClient(Map<String, Object> conf) {
+ @SuppressWarnings("unchecked")
List<String> zkServers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS);
Object port = conf.get(Config.STORM_ZOOKEEPER_PORT);
ZookeeperAuthInfo zkAuthInfo = new ZookeeperAuthInfo(conf);
@@ -106,7 +108,7 @@ public class BlobStoreUtils {
}
// Download missing blobs from potential nimbodes
- public static boolean downloadMissingBlob(Map conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
+ public static boolean downloadMissingBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
throws TTransportException {
NimbusClient client;
ReadableBlobMeta rbm;
@@ -156,7 +158,7 @@ public class BlobStoreUtils {
}
// Download updated blobs from potential nimbodes
- public static boolean downloadUpdatedBlob(Map conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
+ public static boolean downloadUpdatedBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
throws TTransportException {
NimbusClient client;
ClientBlobStore remoteBlobStore;
@@ -215,17 +217,17 @@ public class BlobStoreUtils {
return keyList;
}
- public static void createStateInZookeeper(Map conf, String key, NimbusInfo nimbusInfo) throws TTransportException {
+ public static void createStateInZookeeper(Map<String, Object> conf, String key, NimbusInfo nimbusInfo) throws TTransportException {
ClientBlobStore cb = new NimbusBlobStore();
cb.setClient(conf, new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null));
cb.createStateInZookeeper(key);
}
- public static void updateKeyForBlobStore (Map conf, BlobStore blobStore, CuratorFramework zkClient, String key, NimbusInfo nimbusDetails) {
+ public static void updateKeyForBlobStore (Map<String, Object> conf, BlobStore blobStore, CuratorFramework zkClient, String key, NimbusInfo nimbusDetails) {
try {
// Most of clojure tests currently try to access the blobs using getBlob. Since, updateKeyForBlobStore
// checks for updating the correct version of the blob as a part of nimbus ha before performing any
- // operation on it, there is a neccessity to stub several test cases to ignore this method. It is a valid
+ // operation on it, there is a necessity to stub several test cases to ignore this method. It is a valid
// trade off to return if nimbusDetails which include the details of the current nimbus host port data are
// not initialized as a part of the test. Moreover, this applies to only local blobstore when used along with
// nimbus ha.
@@ -238,6 +240,7 @@ public class BlobStoreUtils {
return;
}
stateInfo = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + "/" + key);
+
LOG.debug("StateInfo for update {}", stateInfo);
Set<NimbusInfo> nimbusInfoList = getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key);
@@ -252,6 +255,9 @@ public class BlobStoreUtils {
LOG.debug("Updating state inside zookeeper for an update");
createStateInZookeeper(conf, key, nimbusDetails);
}
+ } catch (NoNodeException e) {
+ //race condition with a delete
+ return;
} catch (Exception exp) {
throw new RuntimeException(exp);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java b/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java
index c2c62bd..8df981b 100644
--- a/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java
+++ b/storm-core/src/jvm/org/apache/storm/blobstore/LocalFsBlobStore.java
@@ -26,6 +26,7 @@ import org.apache.storm.generated.ReadableBlobMeta;
import org.apache.storm.nimbus.NimbusInfo;
import org.apache.storm.utils.Utils;
+import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.curator.framework.CuratorFramework;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -270,7 +271,12 @@ public class LocalFsBlobStore extends BlobStore {
if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + key) == null) {
return 0;
}
- replicationCount = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + key).size();
+ try {
+ replicationCount = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + key).size();
+ } catch (NoNodeException e) {
+ //Race with delete
+ //If it is not here the replication is 0
+ }
return replicationCount;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java b/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
index 443e471..704c9e5 100644
--- a/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
+++ b/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
@@ -53,6 +53,12 @@ public interface IStormClusterState {
public List<String> activeStorms();
+ /**
+ * Get a storm base for a topology
+ * @param stormId the id of the topology
+ * @param callback something to call if the data changes (best effort)
+ * @return the StormBase or null if it is not alive.
+ */
public StormBase stormBase(String stormId, Runnable callback);
public ClusterWorkerHeartbeat getWorkerHeartbeat(String stormId, String node, Long port);
@@ -183,8 +189,10 @@ public interface IStormClusterState {
Map<String, StormBase> stormBases = new HashMap<>();
for (String topologyId : activeStorms()) {
StormBase base = stormBase(topologyId, null);
- stormBases.put(topologyId, base);
+ if (base != null) { //rece condition with delete
+ stormBases.put(topologyId, base);
+ }
}
return stormBases;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/daemon/nimbus/Nimbus.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/nimbus/Nimbus.java b/storm-core/src/jvm/org/apache/storm/daemon/nimbus/Nimbus.java
index 63d39f8..65c7416 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/nimbus/Nimbus.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/nimbus/Nimbus.java
@@ -749,9 +749,11 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
private static String extractStatusStr(StormBase base) {
String ret = null;
- TopologyStatus status = base.get_status();
- if (status != null) {
- ret = status.name().toUpperCase();
+ if (base != null) {
+ TopologyStatus status = base.get_status();
+ if (status != null) {
+ ret = status.name().toUpperCase();
+ }
}
return ret;
}
@@ -1308,18 +1310,15 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
minReplicationCount, confCount, codeCount, jarCount);
}
- private TopologyDetails readTopologyDetails(String topoId) throws NotAliveException, KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
- StormBase base = stormClusterState.stormBase(topoId, null);
- if (base == null) {
- if (topoId == null) {
- throw new NullPointerException();
- }
- throw new NotAliveException(topoId);
- }
+ private TopologyDetails readTopologyDetails(String topoId, StormBase base) throws KeyNotFoundException,
+ AuthorizationException, IOException, InvalidTopologyException {
+ assert (base != null);
+ assert (topoId != null);
+
BlobStore store = blobStore;
Map<String, Object> topoConf = readTopoConfAsNimbus(topoId, store);
StormTopology topo = readStormTopologyAsNimbus(topoId, store);
- Map<List<Integer>, String> rawExecToComponent = computeExecutorToComponent(topoId);
+ Map<List<Integer>, String> rawExecToComponent = computeExecutorToComponent(topoId, base);
Map<ExecutorDetails, String> executorsToComponent = new HashMap<>();
for (Entry<List<Integer>, String> entry: rawExecToComponent.entrySet()) {
List<Integer> execs = entry.getKey();
@@ -1380,9 +1379,10 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
return ret;
}
- private List<List<Integer>> computeExecutors(String topoId) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
+ private List<List<Integer>> computeExecutors(String topoId, StormBase base) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
BlobStore store = blobStore;
- StormBase base = stormClusterState.stormBase(topoId, null);
+ assert (base != null);
+
Map<String, Integer> compToExecutors = base.get_component_executors();
Map<String, Object> topoConf = readTopoConfAsNimbus(topoId, store);
StormTopology topology = readStormTopologyAsNimbus(topoId, store);
@@ -1405,9 +1405,9 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
return ret;
}
- private Map<List<Integer>, String> computeExecutorToComponent(String topoId) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException {
+ private Map<List<Integer>, String> computeExecutorToComponent(String topoId, StormBase base) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException {
BlobStore store = blobStore;
- List<List<Integer>> executors = computeExecutors(topoId);
+ List<List<Integer>> executors = computeExecutors(topoId, base);
StormTopology topology = readStormTopologyAsNimbus(topoId, store);
Map<String, Object> topoConf = readTopoConfAsNimbus(topoId, store);
Map<Integer, String> taskToComponent = StormCommon.stormTaskInfo(topology, topoConf);
@@ -1418,11 +1418,12 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
return ret;
}
- private Map<String, Set<List<Integer>>> computeTopologyToExecutors(Collection<String> topoIds) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException {
+ private Map<String, Set<List<Integer>>> computeTopologyToExecutors(Map<String, StormBase> bases) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException {
Map<String, Set<List<Integer>>> ret = new HashMap<>();
- if (topoIds != null) {
- for (String topoId: topoIds) {
- ret.put(topoId, new HashSet<>(computeExecutors(topoId)));
+ if (bases != null) {
+ for (Entry<String, StormBase> entry: bases.entrySet()) {
+ String topoId = entry.getKey();
+ ret.put(topoId, new HashSet<>(computeExecutors(topoId, entry.getValue())));
}
}
return ret;
@@ -1579,8 +1580,8 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
private Map<String, SchedulerAssignment> computeNewSchedulerAssignments(Map<String, Assignment> existingAssignments,
- Topologies topologies, String scratchTopologyId) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException {
- Map<String, Set<List<Integer>>> topoToExec = computeTopologyToExecutors(existingAssignments.keySet());
+ Topologies topologies, Map<String, StormBase> bases, String scratchTopologyId) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException {
+ Map<String, Set<List<Integer>>> topoToExec = computeTopologyToExecutors(bases);
updateAllHeartbeats(existingAssignments, topoToExec);
@@ -1641,12 +1642,12 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
return cluster.getAssignments();
}
- private TopologyResources getResourcesForTopology(String topoId) throws NotAliveException, AuthorizationException, InvalidTopologyException, IOException {
+ private TopologyResources getResourcesForTopology(String topoId, StormBase base) throws NotAliveException, AuthorizationException, InvalidTopologyException, IOException {
TopologyResources ret = idToResources.get().get(topoId);
if (ret == null) {
try {
IStormClusterState state = stormClusterState;
- TopologyDetails details = readTopologyDetails(topoId);
+ TopologyDetails details = readTopologyDetails(topoId, base);
double sumOnHeap = 0.0;
double sumOffHeap = 0.0;
double sumCPU = 0.0;
@@ -1720,10 +1721,17 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
IStormClusterState state = stormClusterState;
//read all the topologies
- List<String> topologyIds = state.activeStorms();
+ Map<String, StormBase> bases = state.topologyBases();
Map<String, TopologyDetails> tds = new HashMap<>();
- for (String id: topologyIds) {
- tds.put(id, readTopologyDetails(id));
+ for (Iterator<Entry<String, StormBase>> it = bases.entrySet().iterator(); it.hasNext(); ) {
+ Entry<String, StormBase> entry = it.next();
+ String id = entry.getKey();
+ try {
+ tds.put(id, readTopologyDetails(id, entry.getValue()));
+ } catch (KeyNotFoundException e) {
+ //A race happened and it is probably not running
+ it.remove();
+ }
}
Topologies topologies = new Topologies(tds);
List<String> assignedTopologyIds = state.assignments(null);
@@ -1737,7 +1745,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
}
// make the new assignments for topologies
- Map<String, SchedulerAssignment> newSchedulerAssignments = computeNewSchedulerAssignments(existingAssignments, topologies, scratchTopoId);
+ Map<String, SchedulerAssignment> newSchedulerAssignments = computeNewSchedulerAssignments(existingAssignments, topologies, bases, scratchTopoId);
Map<String, Map<List<Long>, List<Object>>> topologyToExecutorToNodePort = computeNewTopoToExecToNodePort(newSchedulerAssignments, existingAssignments);
for (String id: assignedTopologyIds) {
if (!topologyToExecutorToNodePort.containsKey(id)) {
@@ -1754,8 +1762,10 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
Map<List<Long>, List<Object>> execToNodePort = entry.getValue();
Assignment existingAssignment = existingAssignments.get(topoId);
Set<String> allNodes = new HashSet<>();
- for (List<Object> nodePort: execToNodePort.values()) {
- allNodes.add((String) nodePort.get(0));
+ if (execToNodePort != null) {
+ for (List<Object> nodePort: execToNodePort.values()) {
+ allNodes.add((String) nodePort.get(0));
+ }
}
Map<String, String> allNodeHost = new HashMap<>();
if (existingAssignment != null) {
@@ -2247,7 +2257,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
if (status != null) {
summary.set_sched_status(status);
}
- TopologyResources resources = getResourcesForTopology(topoId);
+ TopologyResources resources = getResourcesForTopology(topoId, base);
if (resources != null) {
summary.set_requested_memonheap(resources.getRequestedMemOnHeap());
summary.set_requested_memoffheap(resources.getRequestedMemOffHeap());
@@ -3271,6 +3281,9 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
try {
getTopologyInfoWithOptsCalls.mark();
CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo");
+ if (common.base == null) {
+ throw new NotAliveException(topoId);
+ }
IStormClusterState state = stormClusterState;
NumErrorsChoice numErrChoice = OR(options.get_num_err_choice(), NumErrorsChoice.ALL);
Map<String, List<ErrorInfo>> errors = new HashMap<>();
@@ -3332,7 +3345,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
if (schedStatus != null) {
topoInfo.set_sched_status(schedStatus);
}
- TopologyResources resources = getResourcesForTopology(topoId);
+ TopologyResources resources = getResourcesForTopology(topoId, common.base);
if (resources != null) {
topoInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
topoInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
@@ -3370,27 +3383,32 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
StormTopology topology = common.topology;
Map<String, Object> topoConf = common.topoConf;
StormBase base = common.base;
- if (assignment == null) {
+ if (base == null) {
throw new NotAliveException(topoId);
}
- Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
- Map<String, String> nodeToHost = assignment.get_node_host();
Map<WorkerSlot, WorkerResources> workerToResources = getWorkerResourcesForTopology(topoId);
+ List<WorkerSummary> workerSummaries = null;
Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
- for (Entry<List<Long>, NodeInfo> entry: execToNodeInfo.entrySet()) {
- NodeInfo ni = entry.getValue();
- List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
- exec2NodePort.put(entry.getKey(), nodePort);
+ if (assignment != null) {
+ Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
+ Map<String, String> nodeToHost = assignment.get_node_host();
+ for (Entry<List<Long>, NodeInfo> entry: execToNodeInfo.entrySet()) {
+ NodeInfo ni = entry.getValue();
+ List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
+ exec2NodePort.put(entry.getKey(), nodePort);
+ }
+
+ workerSummaries = StatsUtil.aggWorkerStats(topoId,
+ topoName,
+ taskToComp,
+ beats,
+ exec2NodePort,
+ nodeToHost,
+ workerToResources,
+ includeSys,
+ true); //this is the topology page, so we know the user is authorized
}
- List<WorkerSummary> workerSummaries = StatsUtil.aggWorkerStats(topoId,
- topoName,
- taskToComp,
- beats,
- exec2NodePort,
- nodeToHost,
- workerToResources,
- includeSys,
- true); //this is the topology page, so we know the user is authorized
+
TopologyPageInfo topoPageInfo = StatsUtil.aggTopoExecsStats(topoId,
exec2NodePort,
taskToComp,
@@ -3412,7 +3430,9 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
commonStats.set_resources_map(setResourcesDefaultIfNotSet(boltResources, entry.getKey(), topoConf));
}
- topoPageInfo.set_workers(workerSummaries);
+ if (workerSummaries != null) {
+ topoPageInfo.set_workers(workerSummaries);
+ }
if (base.is_set_owner()) {
topoPageInfo.set_owner(base.get_owner());
}
@@ -3420,7 +3440,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
if (schedStatus != null) {
topoPageInfo.set_sched_status(schedStatus);
}
- TopologyResources resources = getResourcesForTopology(topoId);
+ TopologyResources resources = getResourcesForTopology(topoId, base);
if (resources != null) {
topoPageInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
topoPageInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
@@ -3525,6 +3545,9 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
try {
getComponentPageInfoCalls.mark();
CommonTopoInfo info = getCommonTopoInfo(topoId, "getComponentPageInfo");
+ if (info.base == null) {
+ throw new NotAliveException(topoId);
+ }
StormTopology topology = info.topology;
Map<String, Object> topoConf = info.topoConf;
Assignment assignment = info.assignment;
@@ -3557,7 +3580,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
compPageInfo.set_topology_name(info.topoName);
compPageInfo.set_errors(stormClusterState.errors(topoId, componentId));
compPageInfo.set_topology_status(extractStatusStr(info.base));
- if (info.base != null && info.base.is_set_component_debug()) {
+ if (info.base.is_set_component_debug()) {
DebugOptions debug = info.base.get_component_debug().get(componentId);
if (debug != null) {
compPageInfo.set_debug_options(debug);
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
index 2dbefcb..6297dda 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.Map;
import org.apache.storm.ProcessSimulator;
-import org.apache.storm.daemon.Shutdownable;
import org.apache.storm.daemon.worker.Worker;
import org.apache.storm.generated.LocalAssignment;
import org.apache.storm.generated.ProfileRequest;
@@ -30,9 +29,6 @@ import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import clojure.java.api.Clojure;
-import clojure.lang.IFn;
-
public class LocalContainer extends Container {
private static final Logger LOG = LoggerFactory.getLogger(LocalContainer.class);
private volatile boolean _isAlive = false;
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/testing/CompletableSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/CompletableSpout.java b/storm-core/src/jvm/org/apache/storm/testing/CompletableSpout.java
new file mode 100644
index 0000000..e9a61f5
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/testing/CompletableSpout.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.testing;
+
+public interface CompletableSpout {
+ /**
+ * @return true if all the tuples have been completed else false.
+ */
+ public boolean isExhausted();
+
+ /**
+ * Cleanup any global state kept
+ */
+ default public void clean() {
+ //NOOP
+ }
+
+ /**
+ * Prepare the spout (globally) before starting the topology
+ */
+ default public void startup() {
+ //NOOP
+ }
+}
[06/10] storm git commit: STORM-1281: LocalCluster,
testing4j and testing.clj to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/testing.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/testing.clj b/storm-core/src/clj/org/apache/storm/testing.clj
deleted file mode 100644
index 60f5a1b..0000000
--- a/storm-core/src/clj/org/apache/storm/testing.clj
+++ /dev/null
@@ -1,766 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-
-(ns org.apache.storm.testing
- (:require [org.apache.storm.daemon
- [local-supervisor :as local-supervisor]
- [common :as common]])
- (:import [org.apache.commons.io FileUtils]
- [org.apache.storm.utils]
- [org.apache.storm.zookeeper Zookeeper]
- [org.apache.storm ProcessSimulator]
- [org.apache.storm.daemon.supervisor Supervisor StandaloneSupervisor SupervisorUtils ReadClusterState]
- [org.apache.storm.executor Executor LocalExecutor]
- [java.util.concurrent.atomic AtomicBoolean])
- (:import [java.io File])
- (:import [java.util HashMap ArrayList])
- (:import [java.util.concurrent.atomic AtomicInteger])
- (:import [java.util.concurrent ConcurrentHashMap])
- (:import [org.apache.storm.utils Time Utils IPredicate RegisteredGlobalState ConfigUtils LocalState StormCommonInstaller])
- (:import [org.apache.storm.tuple Fields Tuple TupleImpl])
- (:import [org.apache.storm.task TopologyContext])
- (:import [org.apache.storm.nimbus ILeaderElector NimbusInfo])
- (:import [org.apache.storm.generated GlobalStreamId Bolt KillOptions])
- (:import [org.apache.storm.testing FeederSpout FixedTupleSpout FixedTuple
- TupleCaptureBolt SpoutTracker BoltTracker NonRichBoltTracker
- TestWordSpout MemoryTransactionalSpout])
- (:import [org.apache.storm.security.auth ThriftServer ThriftConnectionType ReqContext AuthUtils])
- (:import [org.apache.storm.generated NotAliveException AlreadyAliveException StormTopology ErrorInfo
- ExecutorInfo InvalidTopologyException Nimbus$Iface Nimbus$Processor SubmitOptions TopologyInitialStatus
- KillOptions RebalanceOptions ClusterSummary SupervisorSummary TopologySummary TopologyInfo
- ExecutorSummary AuthorizationException GetInfoOptions NumErrorsChoice])
- (:import [org.apache.storm.transactional TransactionalSpoutCoordinator])
- (:import [org.apache.storm.transactional.partitioned PartitionedTransactionalSpoutExecutor])
- (:import [org.apache.storm.tuple Tuple])
- (:import [org.apache.storm Thrift])
- (:import [org.apache.storm Config])
- (:import [org.apache.storm.generated StormTopology])
- (:import [org.apache.storm.task TopologyContext]
- [org.apache.storm.messaging IContext]
- [org.json.simple JSONValue]
- [org.apache.storm.daemon.nimbus Nimbus Nimbus$StandaloneINimbus]
- [org.apache.storm.daemon StormCommon Acker DaemonCommon])
- (:import [org.apache.storm.cluster ZKStateStorage ClusterStateContext StormClusterStateImpl ClusterUtils])
- (:use [org.apache.storm util config log local-state-converter converter])
- (:use [org.apache.storm.internal thrift]))
-
-(defn feeder-spout
- [fields]
- (FeederSpout. (Fields. fields)))
-
-(defn local-temp-path
- []
- (str (System/getProperty "java.io.tmpdir") (if-not (Utils/isOnWindows) "/") (Utils/uuid)))
-
-(defn delete-all
- [paths]
- (dorun
- (for [t paths]
- (if (.exists (File. t))
- (try
- (FileUtils/forceDelete (File. t))
- (catch Exception e
- (log-message (.getMessage e))))))))
-
-(defmacro with-local-tmp
- [[& tmp-syms] & body]
- (let [tmp-paths (mapcat (fn [t] [t `(local-temp-path)]) tmp-syms)]
- `(let [~@tmp-paths]
- (try
- ~@body
- (finally
- (delete-all ~(vec tmp-syms)))))))
-
-(defn start-simulating-time!
- []
- (Time/startSimulating))
-
-(defn stop-simulating-time!
- []
- (Time/stopSimulating))
-
-(defmacro with-simulated-time
- [& body]
- `(try
- (start-simulating-time!)
- ~@body
- (finally
- (stop-simulating-time!))))
-
-(defn advance-time-ms! [ms]
- (Time/advanceTime ms))
-
-(defn advance-time-secs! [secs]
- (advance-time-ms! (* (long secs) 1000)))
-
-(defn set-var-root*
- [avar val]
- (alter-var-root avar (fn [avar] val)))
-
-(defmacro set-var-root
- [var-sym val]
- `(set-var-root* (var ~var-sym) ~val))
-
-(defmacro with-var-roots
- [bindings & body]
- (let [settings (partition 2 bindings)
- tmpvars (repeatedly (count settings) (partial gensym "old"))
- vars (map first settings)
- savevals (vec (mapcat (fn [t v] [t v]) tmpvars vars))
- setters (for [[v s] settings] `(set-var-root ~v ~s))
- restorers (map (fn [v s] `(set-var-root ~v ~s)) vars tmpvars)]
- `(let ~savevals
- ~@setters
- (try
- ~@body
- (finally
- ~@restorers)))))
-
-(defnk add-supervisor
- [cluster-map :ports 2 :conf {} :id nil]
- (let [tmp-dir (local-temp-path)
- port-ids (if (sequential? ports)
- ports
- (doall (repeatedly ports (:port-counter cluster-map))))
- supervisor-conf (merge (:daemon-conf cluster-map)
- conf
- {STORM-LOCAL-DIR tmp-dir
- SUPERVISOR-SLOTS-PORTS port-ids})
- id-fn (if id id (Utils/uuid))
- isupervisor (proxy [StandaloneSupervisor] []
- (generateSupervisorId [] id-fn))
- daemon (local-supervisor/mk-local-supervisor supervisor-conf (:shared-context cluster-map) isupervisor)]
- (swap! (:supervisors cluster-map) conj daemon)
- (swap! (:tmp-dirs cluster-map) conj tmp-dir)
- daemon))
-
-(defn mk-shared-context [conf]
- (if-not (conf STORM-LOCAL-MODE-ZMQ)
- (let [context (org.apache.storm.messaging.local.Context.)]
- (.prepare ^IContext context nil)
- context)))
-
-(defn start-nimbus-daemon [conf nimbus]
- (let [server (ThriftServer. conf (Nimbus$Processor. nimbus)
- ThriftConnectionType/NIMBUS)
- nimbus-thread (Thread. (fn [] (.serve server)))]
- (log-message "Starting Nimbus server...")
- (.start nimbus-thread)
- server))
-
-
-(defn- mk-counter
- ([] (mk-counter 1))
- ([start-val]
- (let [val (atom (dec start-val))]
- (fn [] (swap! val inc)))))
-
-(defnk mock-leader-elector [:is-leader true :leader-name "test-host" :leader-port 9999]
- (let [leader-address (NimbusInfo. leader-name leader-port true)]
- (reify ILeaderElector
- (prepare [this conf] true)
- (isLeader [this] is-leader)
- (addToLeaderLockQueue [this] true)
- (getLeader [this] leader-address)
- (getAllNimbuses [this] `(leader-address))
- (close [this] true))))
-
-(defn mk-nimbus
- [conf inimbus blob-store leader-elector group-mapper cluster-state]
- (Nimbus. conf inimbus cluster-state nil blob-store leader-elector group-mapper))
-
-(defnk mk-mocked-nimbus
- [:daemon-conf {} :inimbus nil :blob-store nil :cluster-state nil
- :leader-elector nil :group-mapper nil :nimbus-daemon false :mk-nimbus mk-nimbus]
- (let [zk-tmp (local-temp-path)
- [zk-port zk-handle] (if-not cluster-state
- (Zookeeper/mkInprocessZookeeper zk-tmp nil))
- leader-elector (or leader-elector (if zk-handle leader-elector (mock-leader-elector)))
- nimbus-tmp (local-temp-path)
- daemon-conf (merge (clojurify-structure (ConfigUtils/readStormConfig))
- {TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true
- ZMQ-LINGER-MILLIS 0
- TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS false
- TOPOLOGY-TRIDENT-BATCH-EMIT-INTERVAL-MILLIS 50
- STORM-CLUSTER-MODE "local"
- BLOBSTORE-SUPERUSER (System/getProperty "user.name")
- BLOBSTORE-DIR nimbus-tmp}
- (if-not cluster-state
- {STORM-ZOOKEEPER-PORT zk-port
- STORM-ZOOKEEPER-SERVERS ["localhost"]})
- daemon-conf)
- nimbus (mk-nimbus
- (assoc daemon-conf STORM-LOCAL-DIR nimbus-tmp)
- (if inimbus inimbus (Nimbus$StandaloneINimbus.))
- blob-store
- leader-elector
- group-mapper
- cluster-state)
- _ (.launchServer nimbus)
- nimbus-thrift-server (if nimbus-daemon (start-nimbus-daemon daemon-conf nimbus) nil)]
- {:nimbus nimbus
- :daemon-conf daemon-conf
- :tmp-dirs (atom [nimbus-tmp zk-tmp])
- :nimbus-thrift-server nimbus-thrift-server
- :zookeeper (if (not-nil? zk-handle) zk-handle)}))
-
-;; returns map containing cluster info
-;; local dir is always overridden in maps
-;; can customize the supervisors (except for ports) by passing in map for :supervisors parameter
-;; if need to customize amt of ports more, can use add-supervisor calls afterwards
-(defnk mk-local-storm-cluster [:supervisors 2 :ports-per-supervisor 3 :daemon-conf {} :inimbus nil :group-mapper nil :supervisor-slot-port-min 1024 :nimbus-daemon false]
- (let [zk-tmp (local-temp-path)
- [zk-port zk-handle] (if-not (contains? daemon-conf STORM-ZOOKEEPER-SERVERS)
- (Zookeeper/mkInprocessZookeeper zk-tmp nil))
- nimbus-tmp (local-temp-path)
- daemon-conf (merge (clojurify-structure (ConfigUtils/readStormConfig))
- {TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true
- ZMQ-LINGER-MILLIS 0
- TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS false
- TOPOLOGY-TRIDENT-BATCH-EMIT-INTERVAL-MILLIS 50
- STORM-CLUSTER-MODE "local"
- BLOBSTORE-SUPERUSER (System/getProperty "user.name")
- BLOBSTORE-DIR nimbus-tmp}
- (if-not (contains? daemon-conf STORM-ZOOKEEPER-SERVERS)
- {STORM-ZOOKEEPER-PORT zk-port
- STORM-ZOOKEEPER-SERVERS ["localhost"]})
- daemon-conf)
- port-counter (mk-counter supervisor-slot-port-min)
- nimbus (mk-nimbus
- (assoc daemon-conf STORM-LOCAL-DIR nimbus-tmp)
- (if inimbus inimbus (Nimbus$StandaloneINimbus.))
- nil
- nil
- group-mapper
- nil)
- _ (.launchServer nimbus)
- context (mk-shared-context daemon-conf)
- nimbus-thrift-server (if nimbus-daemon (start-nimbus-daemon daemon-conf nimbus) nil)
- cluster-map {:nimbus nimbus
- :port-counter port-counter
- :daemon-conf daemon-conf
- :supervisors (atom [])
- :state (ClusterUtils/mkStateStorage daemon-conf nil nil (ClusterStateContext.))
- :storm-cluster-state (ClusterUtils/mkStormClusterState daemon-conf nil (ClusterStateContext.))
- :tmp-dirs (atom [nimbus-tmp zk-tmp])
- :zookeeper (if (not-nil? zk-handle) zk-handle)
- :shared-context context
- :nimbus-thrift-server nimbus-thrift-server}
- supervisor-confs (if (sequential? supervisors)
- supervisors
- (repeat supervisors {}))]
-
- (doseq [sc supervisor-confs]
- (add-supervisor cluster-map :ports ports-per-supervisor :conf sc))
- cluster-map))
-
-(defn get-supervisor [cluster-map supervisor-id]
- (let [pred (reify IPredicate (test [this x] (= (.getId x) supervisor-id)))]
- (Utils/findOne pred @(:supervisors cluster-map))))
-
-(defn remove-first
- [pred aseq]
- (let [[b e] (split-with (complement pred) aseq)]
- (when (empty? e)
- (throw (IllegalArgumentException. "Nothing to remove")))
- (concat b (rest e))))
-
-(defn kill-supervisor [cluster-map supervisor-id]
- (let [finder-fn #(= (.getId %) supervisor-id)
- pred (reify IPredicate (test [this x] (= (.getId x) supervisor-id)))
- supervisors @(:supervisors cluster-map)
- sup (Utils/findOne pred
- supervisors)]
- ;; tmp-dir will be taken care of by shutdown
- (reset! (:supervisors cluster-map) (remove-first finder-fn supervisors))
- (.close sup)))
-
-(defn kill-local-storm-cluster [cluster-map]
- (.shutdown (:nimbus cluster-map))
- (if (not-nil? (:nimbus-thrift-server cluster-map))
- (do
- (log-message "shutting down thrift server")
- (try
- (.stop (:nimbus-thrift-server cluster-map))
- (catch Exception e (log-message "failed to stop thrift")))
- ))
- (if (:state cluster-map) (.close (:state cluster-map)))
- (if (:storm-cluster-state cluster-map) (.disconnect (:storm-cluster-state cluster-map)))
- (if (:supervisors cluster-map)
- (doseq [s @(:supervisors cluster-map)]
- (.shutdownAllWorkers s nil ReadClusterState/THREAD_DUMP_ON_ERROR)
- (.close s)))
- (ProcessSimulator/killAllProcesses)
- (if (not-nil? (:zookeeper cluster-map))
- (do
- (log-message "Shutting down in process zookeeper")
- (Zookeeper/shutdownInprocessZookeeper (:zookeeper cluster-map))
- (log-message "Done shutting down in process zookeeper")))
- (doseq [t @(:tmp-dirs cluster-map)]
- (log-message "Deleting temporary path " t)
- (try
- (Utils/forceDelete t)
- ;; on windows, the host process still holds lock on the logfile
- (catch Exception e (log-message (.getMessage e)))) ))
-
-(def TEST-TIMEOUT-MS
- (let [timeout (System/getenv "STORM_TEST_TIMEOUT_MS")]
- (Integer/parseInt (if timeout timeout "10000"))))
-
-(defmacro while-timeout [timeout-ms condition & body]
- `(let [end-time# (+ (System/currentTimeMillis) ~timeout-ms)]
- (log-debug "Looping until " '~condition)
- (while ~condition
- (when (> (System/currentTimeMillis) end-time#)
- (let [thread-dump# (Utils/threadDump)]
- (log-message "Condition " '~condition " not met in " ~timeout-ms "ms")
- (log-message thread-dump#)
- (throw (AssertionError. (str "Test timed out (" ~timeout-ms "ms) " '~condition)))))
- ~@body)
- (log-debug "Condition met " '~condition)))
-
-(defn wait-for-condition
- ([apredicate]
- (wait-for-condition TEST-TIMEOUT-MS apredicate))
- ([timeout-ms apredicate]
- (while-timeout timeout-ms (not (apredicate))
- (Time/sleep 100))))
-(defn is-supervisor-waiting [^Supervisor supervisor]
- (.isWaiting supervisor))
-
-(defn wait-until-cluster-waiting
- "Wait until the cluster is idle. Should be used with time simulation."
- ([cluster-map] (wait-until-cluster-waiting cluster-map TEST-TIMEOUT-MS))
- ([cluster-map timeout-ms]
- ;; wait until all workers, supervisors, and nimbus is waiting
- (let [supervisors @(:supervisors cluster-map)
- workers (filter (partial instance? DaemonCommon) (clojurify-structure (ProcessSimulator/getAllProcessHandles)))
- daemons (concat
- [(:nimbus cluster-map)]
- ; because a worker may already be dead
- workers)]
- (while-timeout timeout-ms (or (not (every? (memfn isWaiting) daemons))
- (not (every? is-supervisor-waiting supervisors)))
- (Thread/sleep (rand-int 20))
- ;; (doseq [d daemons]
- ;; (if-not ((memfn waiting?) d)
- ;; (println d)))
- ))))
-
-(defn advance-cluster-time
- ([cluster-map secs increment-secs]
- (loop [left secs]
- (when (> left 0)
- (let [diff (min left increment-secs)]
- (advance-time-secs! diff)
- (wait-until-cluster-waiting cluster-map)
- (recur (- left diff))))))
- ([cluster-map secs]
- (advance-cluster-time cluster-map secs 1)))
-
-(defmacro with-mocked-nimbus
- [[nimbus-sym & args] & body]
- `(let [~nimbus-sym (mk-mocked-nimbus ~@args)]
- (try
- ~@body
- (catch Throwable t#
- (log-error t# "Error in cluster")
- (throw t#))
- (finally
- (let [keep-waiting?# (atom true)
- f# (future (while @keep-waiting?# (simulate-wait ~nimbus-sym)))]
- (kill-local-storm-cluster ~nimbus-sym)
- (reset! keep-waiting?# false)
- @f#)))))
-
-(defmacro with-local-cluster
- [[cluster-sym & args] & body]
- `(let [~cluster-sym (mk-local-storm-cluster ~@args)]
- (try
- ~@body
- (catch Throwable t#
- (log-error t# "Error in cluster")
- (throw t#))
- (finally
- (let [keep-waiting?# (atom true)
- f# (future (while @keep-waiting?# (simulate-wait ~cluster-sym)))]
- (kill-local-storm-cluster ~cluster-sym)
- (reset! keep-waiting?# false)
- @f#)))))
-
-(defmacro with-simulated-time-local-cluster
- [& args]
- `(with-simulated-time
- (with-local-cluster ~@args)))
-
-(defmacro with-inprocess-zookeeper
- [port-sym & body]
- `(with-local-tmp [tmp#]
- (let [[~port-sym zks#] (Zookeeper/mkInprocessZookeeper tmp# nil)]
- (try
- ~@body
- (finally
- (Zookeeper/shutdownInprocessZookeeper zks#))))))
-
-(defn submit-local-topology
- [nimbus storm-name conf topology]
- (when-not (Utils/isValidConf conf)
- (throw (IllegalArgumentException. "Topology conf is not json-serializable")))
- (.submitTopology nimbus storm-name nil (JSONValue/toJSONString conf) topology))
-
-(defn submit-local-topology-with-opts
- [nimbus storm-name conf topology submit-opts]
- (when-not (Utils/isValidConf conf)
- (throw (IllegalArgumentException. "Topology conf is not json-serializable")))
- (.submitTopologyWithOpts nimbus storm-name nil (JSONValue/toJSONString conf) topology submit-opts))
-
-(defn find-worker-port
- [supervisor-conf worker-id]
- (let [supervisor-state (ConfigUtils/supervisorState supervisor-conf)
- worker->port (.getApprovedWorkers ^LocalState supervisor-state)]
- (if worker->port (.get worker->port worker-id))))
-
-(defn mk-capture-shutdown-fn
- [capture-atom]
- (fn [supervisorData worker-manager workerId]
- (let [conf (.getConf supervisorData)
- supervisor-id (.getSupervisorId supervisorData)
- port (find-worker-port conf workerId)
- worker-pids (.getWorkerThreadPids supervisorData)
- dead-workers (.getDeadWorkers supervisorData)
- existing (get @capture-atom [supervisor-id port] 0)]
- (log-message "mk-capture-shutdown-fn")
- (swap! capture-atom assoc [supervisor-id port] (inc existing))
- (.shutdownWorker worker-manager supervisor-id workerId worker-pids)
- (if (.cleanupWorker worker-manager workerId)
- (.remove dead-workers workerId)))))
-
-(defnk aggregated-stat
- [cluster-map storm-name stat-key :component-ids nil]
- (let [state (:storm-cluster-state cluster-map)
- nimbus (:nimbus cluster-map)
- storm-id (StormCommon/getStormId state storm-name)
- component->tasks (clojurify-structure (Utils/reverseMap
- (StormCommon/stormTaskInfo
- (.getUserTopology nimbus storm-id)
- (->>
- (.getTopologyConf nimbus storm-id)
- (#(if % (JSONValue/parse %)))
- clojurify-structure))))
- component->tasks (if component-ids
- (select-keys component->tasks component-ids)
- component->tasks)
- task-ids (apply concat (vals component->tasks))
- assignment (clojurify-assignment (.assignmentInfo state storm-id nil))
- taskbeats (.taskbeats state storm-id (:task->node+port assignment))
- heartbeats (dofor [id task-ids] (get taskbeats id))
- stats (dofor [hb heartbeats] (if hb (.get (.get hb "stats") stat-key) 0))]
- (reduce + stats)))
-
-(defn emitted-spout-tuples
- [cluster-map topology storm-name]
- (aggregated-stat
- cluster-map
- storm-name
- "emitted"
- :component-ids (keys (.get_spouts topology))))
-
-(defn transferred-tuples
- [cluster-map storm-name]
- (aggregated-stat cluster-map storm-name "transferred"))
-
-(defn acked-tuples
- [cluster-map storm-name]
- (aggregated-stat cluster-map storm-name "acked"))
-
-(defn simulate-wait
- [cluster-map]
- (if (Time/isSimulating)
- (advance-cluster-time cluster-map 10)
- (Thread/sleep 100)))
-
-(defprotocol CompletableSpout
- (exhausted?
- [this]
- "Whether all the tuples for this spout have been completed.")
- (cleanup
- [this]
- "Cleanup any global state kept")
- (startup
- [this]
- "Prepare the spout (globally) before starting the topology"))
-
-(extend-type FixedTupleSpout
- CompletableSpout
- (exhausted? [this]
- (= (-> this .getSourceTuples count)
- (.getCompleted this)))
- (cleanup [this]
- (.cleanup this))
- (startup [this]))
-
-(extend-type TransactionalSpoutCoordinator
- CompletableSpout
- (exhausted? [this]
- (exhausted? (.getSpout this)))
- (cleanup [this]
- (cleanup (.getSpout this)))
- (startup [this]
- (startup (.getSpout this))))
-
-(extend-type PartitionedTransactionalSpoutExecutor
- CompletableSpout
- (exhausted? [this]
- (exhausted? (.getPartitionedSpout this)))
- (cleanup [this]
- (cleanup (.getPartitionedSpout this)))
- (startup [this]
- (startup (.getPartitionedSpout this))))
-
-(extend-type MemoryTransactionalSpout
- CompletableSpout
- (exhausted? [this]
- (.isExhaustedTuples this))
- (cleanup [this]
- (.cleanup this))
- (startup [this]
- (.startup this)))
-
-(defn spout-objects [spec-map]
- (for [[_ spout-spec] spec-map]
- (-> spout-spec
- .get_spout_object
- (Thrift/deserializeComponentObject))))
-
-(defn capture-topology
- [topology]
- (let [topology (.deepCopy topology)
- spouts (.get_spouts topology)
- bolts (.get_bolts topology)
- all-streams (apply concat
- (for [[id spec] (merge (clojurify-structure spouts)
- (clojurify-structure bolts))]
- (for [[stream info] (.. spec get_common get_streams)]
- [(GlobalStreamId. id stream) (.is_direct info)])))
- capturer (TupleCaptureBolt.)]
- (.set_bolts topology
- (assoc (clojurify-structure bolts)
- (Utils/uuid)
- (Bolt.
- (Thrift/serializeComponentObject capturer)
- (Thrift/prepareComponentCommon (into {} (for [[id direct?] all-streams]
- [id (if direct?
- (Thrift/prepareDirectGrouping)
- (Thrift/prepareGlobalGrouping))]))
- {}
- nil))))
- {:topology topology
- :capturer capturer}))
-
-;; TODO: mock-sources needs to be able to mock out state spouts as well
-;TODO: when translating this function, you should replace the map-val with a proper for loop HERE
-(defnk complete-topology
- [cluster-map topology
- :mock-sources {}
- :storm-conf {}
- :cleanup-state true
- :topology-name nil
- :timeout-ms TEST-TIMEOUT-MS]
- ;; TODO: the idea of mocking for transactional topologies should be done an
- ;; abstraction level above... should have a complete-transactional-topology for this
- (let [{topology :topology capturer :capturer} (capture-topology topology)
- storm-name (or topology-name (str "topologytest-" (Utils/uuid)))
- state (:storm-cluster-state cluster-map)
- spouts (.get_spouts topology)
- replacements (map-val (fn [v]
- (FixedTupleSpout.
- (for [tup v]
- (if (map? tup)
- (FixedTuple. (:stream tup) (:values tup))
- tup))))
- mock-sources)]
- (doseq [[id spout] replacements]
- (let [spout-spec (get spouts id)]
- (.set_spout_object spout-spec (Thrift/serializeComponentObject spout))))
- (doseq [spout (spout-objects spouts)]
- (when-not (extends? CompletableSpout (.getClass spout))
- (throw (RuntimeException. (str "Cannot complete topology unless every spout is a CompletableSpout (or mocked to be); failed by " spout)))))
-
- (doseq [spout (spout-objects spouts)]
- (startup spout))
-
- (submit-local-topology (:nimbus cluster-map) storm-name storm-conf topology)
- (when (Time/isSimulating)
- (advance-cluster-time cluster-map 11))
-
- (let [storm-id (StormCommon/getStormId state storm-name)]
- ;;Give the topology time to come up without using it to wait for the spouts to complete
- (simulate-wait cluster-map)
-
- (while-timeout timeout-ms (not (every? exhausted? (spout-objects spouts)))
- (simulate-wait cluster-map))
-
- (.killTopologyWithOpts (:nimbus cluster-map) storm-name (doto (KillOptions.) (.set_wait_secs 0)))
- (while-timeout timeout-ms (clojurify-assignment (.assignmentInfo state storm-id nil))
- (simulate-wait cluster-map))
- (when cleanup-state
- (doseq [spout (spout-objects spouts)]
- (cleanup spout))))
-
- (if cleanup-state
- (.getAndRemoveResults capturer)
- (.getAndClearResults capturer))))
-
-(defn read-tuples
- ([results component-id stream-id]
- (let [fixed-tuples (get results component-id [])]
- (mapcat
- (fn [ft]
- (if (= stream-id (. ft stream))
- [(vec (. ft values))]))
- fixed-tuples)
- ))
- ([results component-id]
- (read-tuples results component-id Utils/DEFAULT_STREAM_ID)))
-
-(defn multi-set
- "Returns a map of elem to count"
- [aseq]
- (apply merge-with +
- (map #(hash-map % 1) aseq)))
-
-(defn ms=
- [& args]
- (apply = (map multi-set args)))
-
-(def TRACKER-BOLT-ID "+++tracker-bolt")
-
-;; TODO: should override system-topology! and wrap everything there
-(defn mk-tracked-topology
- ([tracked-cluster topology]
- (let [track-id (::track-id tracked-cluster)
- ret (.deepCopy topology)]
- (dofor [[_ bolt] (.get_bolts ret)
- :let [obj (Thrift/deserializeComponentObject (.get_bolt_object bolt))]]
- (.set_bolt_object bolt (Thrift/serializeComponentObject
- (BoltTracker. obj track-id))))
- (dofor [[_ spout] (.get_spouts ret)
- :let [obj (Thrift/deserializeComponentObject (.get_spout_object spout))]]
- (.set_spout_object spout (Thrift/serializeComponentObject
- (SpoutTracker. obj track-id))))
- {:topology ret
- :last-spout-emit (atom 0)
- :cluster tracked-cluster})))
-
-(defn assoc-track-id
- [cluster track-id]
- (assoc cluster ::track-id track-id))
-
-(defn increment-global!
- [id key amt]
- (-> (RegisteredGlobalState/getState id)
- (get key)
- (.addAndGet amt)))
-
-(defn global-amt
- [id key]
- (-> (RegisteredGlobalState/getState id)
- (get key)
- .get))
-
-(defmacro with-tracked-cluster
- [[cluster-sym & cluster-args] & body]
- `(let [id# (Utils/uuid)
- fake-common# (proxy [StormCommon] []
- (makeAckerBoltImpl [] (let [tracker-acker# (NonRichBoltTracker. (Acker.) (String. id#))]
- tracker-acker#)))]
- (with-open [-# (StormCommonInstaller. fake-common#)]
- (RegisteredGlobalState/setState
- id#
- (doto (ConcurrentHashMap.)
- (.put "spout-emitted" (AtomicInteger. 0))
- (.put "transferred" (AtomicInteger. 0))
- (.put "processed" (AtomicInteger. 0))))
- (LocalExecutor/setTrackId id#)
- (with-simulated-time-local-cluster [~cluster-sym ~@cluster-args]
- (let [~cluster-sym (assoc-track-id ~cluster-sym id#)]
- ~@body))
- (LocalExecutor/clearTrackId)
- (RegisteredGlobalState/clearState id#))))
-
-(defn tracked-wait
- "Waits until topology is idle and 'amt' more tuples have been emitted by spouts."
- ([tracked-topology]
- (tracked-wait tracked-topology 1 TEST-TIMEOUT-MS))
- ([tracked-topology amt]
- (tracked-wait tracked-topology amt TEST-TIMEOUT-MS))
- ([tracked-topology amt timeout-ms]
- (let [target (+ amt @(:last-spout-emit tracked-topology))
- track-id (-> tracked-topology :cluster ::track-id)
- waiting? (fn []
- (let [spout-emitted (global-amt track-id "spout-emitted")
- transferred (global-amt track-id "transferred")
- processed (global-amt track-id "processed")]
- (log-message "emitted " spout-emitted " target " target " transferred " transferred " processed " processed)
- (or (not= target spout-emitted)
- (not= transferred
- processed))))]
- (while-timeout timeout-ms (waiting?)
- ;; (println "Spout emitted: " (global-amt track-id "spout-emitted"))
- ;; (println "Processed: " (global-amt track-id "processed"))
- ;; (println "Transferred: " (global-amt track-id "transferred"))
- (advance-time-secs! 1)
- (Thread/sleep (rand-int 200)))
- (reset! (:last-spout-emit tracked-topology) target))))
-
-(defnk test-tuple
- [values
- :stream Utils/DEFAULT_STREAM_ID
- :component "component"
- :fields nil]
- (let [fields (or fields
- (->> (iterate inc 1)
- (take (count values))
- (map #(str "field" %))))
- spout-spec (Thrift/prepareSerializedSpoutDetails
- (TestWordSpout.)
- {stream fields})
- topology (StormTopology. {component spout-spec} {} {})
- context (TopologyContext.
- topology
- (clojurify-structure (ConfigUtils/readStormConfig))
- {(int 1) component}
- {component [(int 1)]}
- {component {stream (Fields. fields)}}
- "test-storm-id"
- nil
- nil
- (int 1)
- nil
- [(int 1)]
- {}
- {}
- (HashMap.)
- (HashMap.)
- (AtomicBoolean. false))]
- (TupleImpl. context values 1 stream)))
-
-(defmacro with-timeout
- [millis unit & body]
- `(let [f# (future ~@body)]
- (try
- (.get f# ~millis ~unit)
- (finally (future-cancel f#)))))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/testing4j.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/testing4j.clj b/storm-core/src/clj/org/apache/storm/testing4j.clj
deleted file mode 100644
index 41f8258..0000000
--- a/storm-core/src/clj/org/apache/storm/testing4j.clj
+++ /dev/null
@@ -1,185 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.testing4j
- (:import [java.util Map List Collection ArrayList])
- (:require [org.apache.storm [LocalCluster :as LocalCluster]])
- (:import [org.apache.storm Config ILocalCluster LocalCluster])
- (:import [org.apache.storm.generated StormTopology])
- (:import [org.apache.storm.testing TestJob MockedSources TrackedTopology
- MkClusterParam CompleteTopologyParam MkTupleParam])
- (:import [org.apache.storm.utils Utils])
- (:use [org.apache.storm testing util log])
- (:gen-class
- :name org.apache.storm.Testing
- :methods [^:static [completeTopology
- [org.apache.storm.ILocalCluster org.apache.storm.generated.StormTopology
- org.apache.storm.testing.CompleteTopologyParam]
- java.util.Map]
- ^:static [completeTopology
- [org.apache.storm.ILocalCluster org.apache.storm.generated.StormTopology]
- java.util.Map]
- ^:static [withSimulatedTime [Runnable] void]
- ^:static [withLocalCluster [org.apache.storm.testing.TestJob] void]
- ^:static [withLocalCluster [org.apache.storm.testing.MkClusterParam org.apache.storm.testing.TestJob] void]
- ^:static [getLocalCluster [java.util.Map] org.apache.storm.ILocalCluster]
- ^:static [withSimulatedTimeLocalCluster [org.apache.storm.testing.TestJob] void]
- ^:static [withSimulatedTimeLocalCluster [org.apache.storm.testing.MkClusterParam org.apache.storm.testing.TestJob] void]
- ^:static [withTrackedCluster [org.apache.storm.testing.TestJob] void]
- ^:static [withTrackedCluster [org.apache.storm.testing.MkClusterParam org.apache.storm.testing.TestJob] void]
- ^:static [readTuples [java.util.Map String String] java.util.List]
- ^:static [readTuples [java.util.Map String] java.util.List]
- ^:static [mkTrackedTopology [org.apache.storm.ILocalCluster org.apache.storm.generated.StormTopology] org.apache.storm.testing.TrackedTopology]
- ^:static [trackedWait [org.apache.storm.testing.TrackedTopology] void]
- ^:static [trackedWait [org.apache.storm.testing.TrackedTopology Integer] void]
- ^:static [trackedWait [org.apache.storm.testing.TrackedTopology Integer Integer] void]
- ^:static [advanceClusterTime [org.apache.storm.ILocalCluster Integer Integer] void]
- ^:static [advanceClusterTime [org.apache.storm.ILocalCluster Integer] void]
- ^:static [multiseteq [java.util.Collection java.util.Collection] boolean]
- ^:static [multiseteq [java.util.Map java.util.Map] boolean]
- ^:static [testTuple [java.util.List] org.apache.storm.tuple.Tuple]
- ^:static [testTuple [java.util.List org.apache.storm.testing.MkTupleParam] org.apache.storm.tuple.Tuple]]))
-
-(defn -completeTopology
- ([^ILocalCluster cluster ^StormTopology topology ^CompleteTopologyParam completeTopologyParam]
- (let [mocked-sources (or (-> completeTopologyParam .getMockedSources .getData) {})
- storm-conf (or (.getStormConf completeTopologyParam) {})
- cleanup-state (or (.getCleanupState completeTopologyParam) true)
- topology-name (.getTopologyName completeTopologyParam)
- timeout-ms (or (.getTimeoutMs completeTopologyParam) TEST-TIMEOUT-MS)]
- (complete-topology (.getState cluster) topology
- :mock-sources mocked-sources
- :storm-conf storm-conf
- :cleanup-state cleanup-state
- :topology-name topology-name
- :timeout-ms timeout-ms)))
- ([^ILocalCluster cluster ^StormTopology topology]
- (-completeTopology cluster topology (CompleteTopologyParam.))))
-
-
-(defn -withSimulatedTime
- [^Runnable code]
- (with-simulated-time
- (.run code)))
-
-(defmacro with-cluster
- [cluster-type mkClusterParam code]
- `(let [supervisors# (or (.getSupervisors ~mkClusterParam) 2)
- ports-per-supervisor# (or (.getPortsPerSupervisor ~mkClusterParam) 3)
- daemon-conf# (or (.getDaemonConf ~mkClusterParam) {})
- nimbus-daemon# (.isNimbusDaemon ~mkClusterParam)]
- (~cluster-type [cluster# :supervisors supervisors#
- :ports-per-supervisor ports-per-supervisor#
- :daemon-conf daemon-conf#
- :nimbus-daemon nimbus-daemon#]
- (let [cluster# (LocalCluster. cluster#)]
- (.run ~code cluster#)))))
-
-(defn -withLocalCluster
- ([^MkClusterParam mkClusterParam ^TestJob code]
- (with-cluster with-local-cluster mkClusterParam code))
- ([^TestJob code]
- (-withLocalCluster (MkClusterParam.) code)))
-
-(defn -getLocalCluster
- ([^Map clusterConf]
- (let [daemon-conf (get-in clusterConf ["daemon-conf"] {})
- supervisors (get-in clusterConf ["supervisors"] 2)
- ports-per-supervisor (get-in clusterConf ["ports-per-supervisor"] 3)
- inimbus (get-in clusterConf ["inimbus"] nil)
- supervisor-slot-port-min (get-in clusterConf ["supervisor-slot-port-min"] 1024)
- nimbus-daemon (get-in clusterConf ["nimbus-daemon"] false)
- local-cluster-map (mk-local-storm-cluster :supervisors supervisors
- :ports-per-supervisor ports-per-supervisor
- :daemon-conf daemon-conf
- :inimbus inimbus
- :supervisor-slot-port-min supervisor-slot-port-min
- :nimbus-daemon nimbus-daemon
- )]
- (LocalCluster. local-cluster-map))))
-
-(defn -withSimulatedTimeLocalCluster
- ([^MkClusterParam mkClusterParam ^TestJob code]
- (with-cluster with-simulated-time-local-cluster mkClusterParam code))
- ([^TestJob code]
- (-withSimulatedTimeLocalCluster (MkClusterParam.) code)))
-
-(defn -withTrackedCluster
- ([^MkClusterParam mkClusterParam ^TestJob code]
- (with-cluster with-tracked-cluster mkClusterParam code))
- ([^TestJob code]
- (-withTrackedCluster (MkClusterParam.) code)))
-
-(defn- find-tuples
- [^List fixed-tuples ^String stream]
- (let [ret (ArrayList.)]
- (doseq [fixed-tuple fixed-tuples]
- (if (= (.stream fixed-tuple) stream)
- (.add ret (.values fixed-tuple))))
- ret))
-
-(defn -readTuples
- ([^Map result ^String componentId ^String streamId]
- (let [stream-result (.get result componentId)
- ret (if stream-result
- (find-tuples stream-result streamId)
- [])]
- ret))
- ([^Map result ^String componentId]
- (-readTuples result componentId Utils/DEFAULT_STREAM_ID)))
-
-(defn -mkTrackedTopology
- [^ILocalCluster trackedCluster ^StormTopology topology]
- (-> (mk-tracked-topology (.getState trackedCluster) topology)
- (TrackedTopology.)))
-
-(defn -trackedWait
- ([^TrackedTopology trackedTopology ^Integer amt ^Integer timeout-ms]
- (tracked-wait trackedTopology amt timeout-ms))
- ([^TrackedTopology trackedTopology ^Integer amt]
- (tracked-wait trackedTopology amt))
- ([^TrackedTopology trackedTopology]
- (-trackedWait trackedTopology 1)))
-
-(defn -advanceClusterTime
- ([^ILocalCluster cluster ^Integer secs ^Integer step]
- (advance-cluster-time (.getState cluster) secs step))
- ([^ILocalCluster cluster ^Integer secs]
- (-advanceClusterTime cluster secs 1)))
-
-(defn- multiseteq
- [^Object obj1 ^Object obj2]
- (let [obj1 (clojurify-structure obj1)
- obj2 (clojurify-structure obj2)]
- (ms= obj1 obj2)))
-
-(defn -multiseteq
- [^Collection coll1 ^Collection coll2]
- (multiseteq coll1 coll2))
-
-(defn -multiseteq
- [^Map coll1 ^Map coll2]
- (multiseteq coll1 coll2))
-
-(defn -testTuple
- ([^List values]
- (-testTuple values nil))
- ([^List values ^MkTupleParam param]
- (if (nil? param)
- (test-tuple values)
- (let [stream (or (.getStream param) Utils/DEFAULT_STREAM_ID)
- component (or (.getComponent param) "component")
- fields (.getFields param)]
- (test-tuple values :stream stream :component component :fields fields)))))
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/trident/testing.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/trident/testing.clj b/storm-core/src/clj/org/apache/storm/trident/testing.clj
deleted file mode 100644
index ba50649..0000000
--- a/storm-core/src/clj/org/apache/storm/trident/testing.clj
+++ /dev/null
@@ -1,78 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.trident.testing
- (:import [org.apache.storm.trident.testing FeederBatchSpout FeederCommitterBatchSpout MemoryMapState MemoryMapState$Factory TuplifyArgs])
- (:import [org.apache.storm LocalDRPC])
- (:import [org.apache.storm.tuple Fields])
- (:import [org.apache.storm.generated KillOptions]
- [org.json.simple JSONValue])
- (:require [org.apache.storm [testing :as t]])
- (:use [org.apache.storm util])
- )
-
-(defn local-drpc []
- (LocalDRPC.))
-
-(defn exec-drpc [^LocalDRPC drpc function-name args]
- (if-let [res (.execute drpc function-name args)]
- (clojurify-structure (JSONValue/parse res))))
-
-(defn exec-drpc-tuples [^LocalDRPC drpc function-name tuples]
- (exec-drpc drpc function-name (JSONValue/toJSONString tuples)))
-
-(defn feeder-spout [fields]
- (FeederBatchSpout. fields))
-
-(defn feeder-committer-spout [fields]
- (FeederCommitterBatchSpout. fields))
-
-(defn feed [feeder tuples]
- (.feed feeder tuples))
-
-(defn fields [& fields]
- (Fields. fields))
-
-(defn memory-map-state []
- (MemoryMapState$Factory.))
-
-(defmacro with-drpc [[drpc] & body]
- `(let [~drpc (org.apache.storm.LocalDRPC.)]
- ~@body
- (.shutdown ~drpc)
- ))
-
-(defn with-topology* [cluster storm-topo body-fn]
- (t/submit-local-topology (:nimbus cluster) "tester" {} storm-topo)
- (body-fn)
- (.killTopologyWithOpts (:nimbus cluster) "tester" (doto (KillOptions.) (.set_wait_secs 0))))
-
-(defmacro with-topology [[cluster topo storm-topo] & body]
- `(let [~storm-topo (.build ~topo)]
- (with-topology* ~cluster ~storm-topo (fn [] ~@body))))
-
-(defn bootstrap-imports []
- (import 'org.apache.storm.LocalDRPC)
- (import 'org.apache.storm.trident.TridentTopology)
- (import '[org.apache.storm.trident.operation.builtin Count Sum Equals MapGet Debug FilterNull FirstN TupleCollectionGet])
- )
-
-(defn drpc-tuples-input [topology function-name drpc outfields]
- (-> topology
- (.newDRPCStream function-name drpc)
- (.each (fields "args") (TuplifyArgs.) outfields)
- ))
-
-
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/core.clj b/storm-core/src/clj/org/apache/storm/ui/core.clj
index 5465b92..4cc8037 100644
--- a/storm-core/src/clj/org/apache/storm/ui/core.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/core.clj
@@ -22,7 +22,7 @@
ring.middleware.multipart-params.temp-file)
(:use [ring.middleware.json :only [wrap-json-params]])
(:use [hiccup core page-helpers])
- (:use [org.apache.storm config util log converter])
+ (:use [org.apache.storm config util log])
(:use [org.apache.storm.ui helpers])
(:import [org.apache.storm.utils Time]
[org.apache.storm.generated NimbusSummary]
@@ -309,7 +309,7 @@
bolt-summs (filter (partial bolt-summary? topology) execs)
spout-comp-summs (group-by-comp spout-summs)
bolt-comp-summs (group-by-comp bolt-summs)
- ;TODO: when translating this function, you should replace the filter-val with a proper for loop + if condition HERE
+ ;TODO: when translating this function, you should replace the filter-key with a proper for loop + if condition HERE
bolt-comp-summs (filter-key (mk-include-sys-fn include-sys?)
bolt-comp-summs)]
(visualization-data
@@ -582,7 +582,7 @@
bolt-executor-summaries (filter (partial bolt-summary? storm-topology) (.get_executors topology-info))
spout-comp-id->executor-summaries (group-by-comp spout-executor-summaries)
bolt-comp-id->executor-summaries (group-by-comp bolt-executor-summaries)
- ;TODO: when translating this function, you should replace the filter-val with a proper for loop + if condition HERE
+ ;TODO: when translating this function, you should replace the filter-key with a proper for loop + if condition HERE
bolt-comp-id->executor-summaries (filter-key (mk-include-sys-fn include-sys?) bolt-comp-id->executor-summaries)
id->spout-spec (.get_spouts storm-topology)
id->bolt (.get_bolts storm-topology)
@@ -994,6 +994,14 @@
(.get_exec_stats info))}
(-> info .get_errors (component-errors topology-id secure?))))
+(defn clojurify-profile-request
+ [^ProfileRequest request]
+ (when request
+ {:host (.get_node (.get_nodeInfo request))
+ :port (first (.get_port (.get_nodeInfo request)))
+ :action (.get_action request)
+ :timestamp (.get_time_stamp request)}))
+
(defn get-active-profile-actions
[nimbus topology-id component]
(let [profile-actions (.getComponentPendingProfileActions nimbus
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/clj/org/apache/storm/util.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/util.clj b/storm-core/src/clj/org/apache/storm/util.clj
index bd76022..f0388a7 100644
--- a/storm-core/src/clj/org/apache/storm/util.clj
+++ b/storm-core/src/clj/org/apache/storm/util.clj
@@ -48,24 +48,6 @@
(:require [ring.util.codec :as codec])
(:use [org.apache.storm log]))
-(defmacro defalias
- "Defines an alias for a var: a new var with the same root binding (if
- any) and similar metadata. The metadata of the alias is its initial
- metadata (as provided by def) merged into the metadata of the original."
- ([name orig]
- `(do
- (alter-meta!
- (if (.hasRoot (var ~orig))
- (def ~name (.getRawRoot (var ~orig)))
- (def ~name))
- ;; When copying metadata, disregard {:macro false}.
- ;; Workaround for http://www.assembla.com/spaces/clojure/tickets/273
- #(conj (dissoc % :macro)
- (apply dissoc (meta (var ~orig)) (remove #{:macro} (keys %)))))
- (var ~name)))
- ([name orig doc]
- (list `defalias (with-meta name (assoc (meta name) :doc doc)) orig)))
-
;; name-with-attributes by Konrad Hinsen:
(defn name-with-attributes
"To be used in macro definitions.
@@ -121,30 +103,6 @@
(if (not tc#) (log-error t# "Exception did not match " ~klass))
tc#))))
-(defmacro forcat
- [[args aseq] & body]
- `(mapcat (fn [~args]
- ~@body)
- ~aseq))
-
-(defmacro try-cause
- [& body]
- (let [checker (fn [form]
- (or (not (sequential? form))
- (not= 'catch (first form))))
- [code guards] (split-with checker body)
- error-local (gensym "t")
- guards (forcat [[_ klass local & guard-body] guards]
- `((Utils/exceptionCauseIsInstanceOf ~klass ~error-local)
- (let [~local ~error-local]
- ~@guard-body
- )))]
- `(try ~@code
- (catch Throwable ~error-local
- (cond ~@guards
- true (throw ~error-local)
- )))))
-
(defn clojurify-structure
[s]
(if s
@@ -178,85 +136,16 @@
;TODO: We're keeping this function around until all the code using it is properly tranlated to java
;TODO: by properly having the for loop IN THE JAVA FUNCTION that originally used this function.
-(defn filter-val
- [afn amap]
- (into {} (filter (fn [[k v]] (afn v)) amap)))
-
-;TODO: We're keeping this function around until all the code using it is properly tranlated to java
-;TODO: by properly having the for loop IN THE JAVA FUNCTION that originally used this function.
(defn filter-key
[afn amap]
(into {} (filter (fn [[k v]] (afn k)) amap)))
-;TODO: We're keeping this function around until all the code using it is properly tranlated to java
-;TODO: by properly having the for loop IN THE JAVA FUNCTION that originally used this function.
-(defn map-key
- [afn amap]
- (into {} (for [[k v] amap] [(afn k) v])))
-
;TODO: Once all the other clojure functions (100+ locations) are translated to java, this function becomes moot.
(def not-nil? (complement nil?))
(defmacro dofor [& body]
`(doall (for ~@body)))
-;; The following two will go away when worker, task, executor go away.
-(defn assoc-apply-self [curr key afn]
- (assoc curr key (afn curr)))
-
-; These seven following will go away later. To be replaced by idiomatic java.
-(defmacro recursive-map
- [& forms]
- (->> (partition 2 forms)
- (map (fn [[key form]] `(assoc-apply-self ~key (fn [~'<>] ~form))))
- (concat `(-> {}))))
-
-(defmacro fast-list-iter
- [pairs & body]
- (let [pairs (partition 2 pairs)
- lists (map second pairs)
- elems (map first pairs)
- iters (map (fn [_] (gensym)) lists)
- bindings (->> (map (fn [i l] (let [lg (gensym)] [lg l i `(if ~lg (.iterator ~lg))])) iters lists)
- (apply concat))
- tests (map (fn [i] `(and ~i (.hasNext ^Iterator ~i))) iters)
- assignments (->> (map (fn [e i] [e `(.next ^Iterator ~i)]) elems iters)
- (apply concat))]
- `(let [~@bindings]
- (while (and ~@tests)
- (let [~@assignments]
- ~@body)))))
-
-(defmacro fast-list-for
- [[e alist] & body]
- `(let [ret# (ArrayList.)]
- (fast-list-iter [~e ~alist]
- (.add ret# (do ~@body)))
- ret#))
-
-(defmacro fast-map-iter
- [[bind amap] & body]
- `(let [iter# (if ~amap (.. ^Map ~amap entrySet iterator))]
- (while (and iter# (.hasNext ^Iterator iter#))
- (let [entry# (.next ^Iterator iter#)
- ~bind [(.getKey ^Map$Entry entry#) (.getValue ^Map$Entry entry#)]]
- ~@body))))
-
-(defn fast-group-by
- [afn alist]
- (let [ret (HashMap.)]
- (fast-list-iter
- [e alist]
- (let [key (afn e)
- ^List curr (let [curr (.get ret key)]
- (if curr
- curr
- (let [default (ArrayList.)]
- (.put ret key default)
- default)))]
- (.add curr e)))
- ret))
-
(defmacro -<>
([x] x)
([x form] (if (seq? form)
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/ILocalCluster.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/ILocalCluster.java b/storm-core/src/jvm/org/apache/storm/ILocalCluster.java
index 05b55e7..018b8be 100644
--- a/storm-core/src/jvm/org/apache/storm/ILocalCluster.java
+++ b/storm-core/src/jvm/org/apache/storm/ILocalCluster.java
@@ -17,33 +17,186 @@
*/
package org.apache.storm;
-import org.apache.storm.generated.AlreadyAliveException;
+import java.util.Map;
+
+import org.apache.storm.cluster.IStormClusterState;
import org.apache.storm.generated.ClusterSummary;
-import org.apache.storm.generated.InvalidTopologyException;
+import org.apache.storm.generated.Credentials;
import org.apache.storm.generated.KillOptions;
-import org.apache.storm.generated.SubmitOptions;
-import org.apache.storm.generated.NotAliveException;
import org.apache.storm.generated.RebalanceOptions;
import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.SubmitOptions;
import org.apache.storm.generated.TopologyInfo;
-import org.apache.storm.generated.Credentials;
-
-import java.util.Map;
+import org.apache.storm.testing.TrackedTopology;
+import org.apache.thrift.TException;
+/**
+ * This is here mostly for backwards compatibility.
+ * Please see {@link org.apache.storm.LocalCluster} for
+ * more details on testing a Storm Topology.
+ */
+public interface ILocalCluster extends AutoCloseable {
+ /**
+ * Close this class to kill the topology.
+ * This is here mostly for backwards compatibility.
+ * Please see {@link org.apache.storm.LocalCluster.LocalTopology} for
+ * more details on testing a Storm Topology.
+ */
+ public interface ILocalTopology extends AutoCloseable {
-public interface ILocalCluster {
- void submitTopology(String topologyName, Map conf, StormTopology topology) throws AlreadyAliveException, InvalidTopologyException;
- void submitTopologyWithOpts(String topologyName, Map conf, StormTopology topology, SubmitOptions submitOpts) throws AlreadyAliveException, InvalidTopologyException;
- void uploadNewCredentials(String topologyName, Credentials creds);
- void killTopology(String topologyName) throws NotAliveException;
- void killTopologyWithOpts(String name, KillOptions options) throws NotAliveException;
- void activate(String topologyName) throws NotAliveException;
- void deactivate(String topologyName) throws NotAliveException;
- void rebalance(String name, RebalanceOptions options) throws NotAliveException;
+ };
+
+ /**
+ * Submit a topology to be run in local mode
+ * @param topologyName the name of the topology to use
+ * @param conf the config for the topology
+ * @param topology the topology itself.
+ * @return an AutoCloseable that will kill the topology.
+ * @throws TException on any error from nimbus
+ */
+ ILocalTopology submitTopology(String topologyName, Map<String, Object> conf, StormTopology topology) throws TException;
+
+ /**
+ * Submit a topology to be run in local mode
+ * @param topologyName the name of the topology to use
+ * @param conf the config for the topology
+ * @param topology the topology itself.
+ * @param submitOpts options for topology
+ * @return an AutoCloseable that will kill the topology.
+ * @throws TException on any error from nimbus
+ */
+ ILocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, StormTopology topology, SubmitOptions submitOpts) throws TException;
+
+ /**
+ * Submit a tracked topology to be run in local mode
+ * @param topologyName the name of the topology to use
+ * @param conf the config for the topology
+ * @param topology the topology itself.
+ * @return an AutoCloseable that will kill the topology.
+ * @throws TException on any error from nimbus
+ */
+ ILocalTopology submitTopology(String topologyName, Map<String, Object> conf, TrackedTopology topology) throws TException;
+
+ /**
+ * Submit a tracked topology to be run in local mode
+ * @param topologyName the name of the topology to use
+ * @param conf the config for the topology
+ * @param topology the topology itself.
+ * @param submitOpts options for topology
+ * @return an AutoCloseable that will kill the topology.
+ * @throws TException on any error from nimbus
+ */
+ ILocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, TrackedTopology topology, SubmitOptions submitOpts) throws TException;
+
+ /**
+ * Upload new credentials to a topology.
+ * @param topologyName the name of the topology
+ * @param creds the new credentails for the topology.
+ * @throws TException on any error from nimbus
+ */
+ void uploadNewCredentials(String topologyName, Credentials creds) throws TException;
+
+ /**
+ * Kill a topology (if you are not using ILocalTopology)
+ * @param topologyName the name of the topology
+ * @throws TException on any error from nimbus
+ */
+ void killTopology(String topologyName) throws TException;
+
+ /**
+ * Kill a topology (if you are not using ILocalTopology)
+ * @param topologyName the name of the topology
+ * @param options for how to kill the topology
+ * @throws TException on any error from nimbus
+ */
+ void killTopologyWithOpts(String name, KillOptions options) throws TException;
+
+ /**
+ * Activate a topology
+ * @param topologyName the name of the topology to activate
+ * @throws TException on any error from nimbus
+ */
+ void activate(String topologyName) throws TException;
+
+ /**
+ * Deactivate a topology
+ * @param topologyName the name of the topology to deactivate
+ * @throws TException on any error from nimbus
+ */
+ void deactivate(String topologyName) throws TException;
+
+ /**
+ * Rebalance a topology
+ * @param name the name of the topology
+ * @param options options for rebalanceing the topology.
+ * @throws TException on any error from nimbus
+ */
+ void rebalance(String name, RebalanceOptions options) throws TException;
+
+ /**
+ * Shut down the cluster.
+ * @deprecated use {@link #close()} instead.
+ */
+ @Deprecated
void shutdown();
- String getTopologyConf(String id);
- StormTopology getTopology(String id);
- ClusterSummary getClusterInfo();
- TopologyInfo getTopologyInfo(String id);
- Map getState();
+
+ /**
+ * The config of a topology as a JSON string
+ * @param id the id of the topology (not the name)
+ * @return The config of a topology as a JSON string
+ * @throws TException on any error from nimbus
+ */
+ String getTopologyConf(String id) throws TException;
+
+ /**
+ * Get the compiled storm topology
+ * @param id the id of the topology (not the name)
+ * @return the compiled storm topology
+ * @throws TException on any error from nimbus
+ */
+ StormTopology getTopology(String id) throws TException;
+
+ /**
+ * @return a summary of the current state of the cluster
+ * @throws TException on any error from nimbus
+ */
+ ClusterSummary getClusterInfo() throws TException;
+
+ /**
+ * Get the state of a topology
+ * @param id the id of the topology (not the name)
+ * @return the state of a topology
+ * @throws TException on any error from nimbus
+ */
+ TopologyInfo getTopologyInfo(String id) throws TException;
+
+ /**
+ * This is intended for internal testing only.
+ * @return an internal class that holds the state of the cluster.
+ */
+ IStormClusterState getClusterState();
+
+ /**
+ * Advance the cluster time when the cluster is using SimulatedTime.
+ * This is intended for internal testing only.
+ * @param secs the number of seconds to advance time
+ * @throws InterruptedException
+ */
+ void advanceClusterTime(int secs) throws InterruptedException;
+
+ /**
+ * Advance the cluster time when the cluster is using SimulatedTime.
+ * This is intended for internal testing only.
+ * @param secs the number of seconds to advance time
+ * @param steps the number of steps we should take when advancing simulated time
+ * @throws InterruptedException
+ */
+ void advanceClusterTime(int secs, int step) throws InterruptedException;
+
+ /**
+ * If the cluster is tracked get the id for the tracked cluster.
+ * This is intended for internal testing only.
+ * @return the id of the tracked cluster
+ */
+ String getTrackedId();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4efcc996/storm-core/src/jvm/org/apache/storm/ILocalDRPC.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/ILocalDRPC.java b/storm-core/src/jvm/org/apache/storm/ILocalDRPC.java
index 8b0a9db..68b60d6 100644
--- a/storm-core/src/jvm/org/apache/storm/ILocalDRPC.java
+++ b/storm-core/src/jvm/org/apache/storm/ILocalDRPC.java
@@ -22,6 +22,15 @@ import org.apache.storm.generated.DistributedRPC;
import org.apache.storm.generated.DistributedRPCInvocations;
-public interface ILocalDRPC extends DistributedRPC.Iface, DistributedRPCInvocations.Iface, Shutdownable {
- public String getServiceId();
+public interface ILocalDRPC extends DistributedRPC.Iface, DistributedRPCInvocations.Iface, Shutdownable, AutoCloseable {
+ /**
+ * Get the ID of the service. This is used internally if multiple local DRPC clusters are in use at one time.
+ */
+ public String getServiceId();
+
+ /**
+ * @deprecated use {@link #close()} instead
+ */
+ @Deprecated
+ public void shutdown();
}