You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/08/24 15:52:14 UTC
[29/50] [abbrv] storm git commit: Merge remote-tracking branch
'upstream/master' into STORM-166
Merge remote-tracking branch 'upstream/master' into STORM-166
Conflicts:
storm-core/src/clj/backtype/storm/daemon/nimbus.clj
storm-core/src/clj/backtype/storm/ui/core.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/23ed5375
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/23ed5375
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/23ed5375
Branch: refs/heads/master
Commit: 23ed53754c47a2170e58b178beb1544616f0f08a
Parents: 63f29f3 8036109
Author: Parth Brahmbhatt <br...@gmail.com>
Authored: Thu Feb 12 16:44:06 2015 -0800
Committer: Parth Brahmbhatt <br...@gmail.com>
Committed: Thu Feb 12 16:44:06 2015 -0800
----------------------------------------------------------------------
.gitignore | 1 +
CHANGELOG.md | 25 +-
README.markdown | 13 +-
SECURITY.md | 14 +-
STORM-UI-REST-API.md | 21 +
bin/storm | 36 +-
bin/storm-config.cmd | 32 +-
bin/storm.cmd | 15 +-
conf/defaults.yaml | 3 +-
dev-tools/github/__init__.py | 11 +
dev-tools/jira-github-join.py | 4 +-
dev-tools/storm-merge.py | 31 +
docs/README.md | 9 +
docs/documentation/Common-patterns.md | 14 +-
docs/documentation/Concepts.md | 13 +-
docs/documentation/Contributing-to-Storm.md | 2 +-
docs/documentation/Home.md | 2 +-
docs/documentation/Multilang-protocol.md | 4 +-
docs/documentation/Powered-By.md | 15 +-
docs/downloads.html | 3 +
.../storm/starter/SkewedRollingTopWords.java | 134 +++
.../storm/starter/bolt/RollingCountAggBolt.java | 78 ++
.../hbase/trident/state/HBaseMapState.java | 2 +-
.../storm/hbase/trident/state/HBaseState.java | 2 +-
external/storm-kafka/README.md | 24 +-
external/storm-kafka/pom.xml | 21 +-
.../ExponentialBackoffMsgRetryManager.java | 167 ++++
.../jvm/storm/kafka/FailedMsgRetryManager.java | 26 +
.../src/jvm/storm/kafka/PartitionManager.java | 42 +-
.../src/jvm/storm/kafka/SpoutConfig.java | 8 +
.../ExponentialBackoffMsgRetryManagerTest.java | 194 ++++
pom.xml | 19 +-
storm-buildtools/storm-maven-plugins/pom.xml | 81 ++
.../storm/maven/plugin/util/CommandExec.java | 89 ++
.../plugin/versioninfo/VersionInfoMojo.java | 304 ++++++
storm-core/pom.xml | 41 +-
storm-core/src/clj/backtype/storm/cluster.clj | 38 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 5 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 41 +-
.../clj/backtype/storm/daemon/supervisor.clj | 5 +-
.../src/clj/backtype/storm/daemon/worker.clj | 6 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 39 +-
storm-core/src/clj/backtype/storm/util.clj | 11 +-
storm-core/src/jvm/backtype/storm/Config.java | 6 +
.../coordination/BatchSubtopologyBuilder.java | 11 +
.../storm/drpc/LinearDRPCInputDeclarer.java | 5 +-
.../storm/drpc/LinearDRPCTopologyBuilder.java | 13 +-
.../storm/generated/GetInfoOptions.java | 350 +++++++
.../jvm/backtype/storm/generated/Nimbus.java | 974 +++++++++++++++++++
.../storm/generated/NumErrorsChoice.java | 64 ++
.../storm/grouping/PartialKeyGrouping.java | 82 ++
.../backtype/storm/messaging/netty/Server.java | 2 +
.../backtype/storm/topology/InputDeclarer.java | 3 +
.../storm/topology/TopologyBuilder.java | 11 +
.../TransactionalTopologyBuilder.java | 13 +-
.../src/jvm/backtype/storm/utils/Monitor.java | 8 +-
.../jvm/backtype/storm/utils/VersionInfo.java | 131 +++
.../topology/TridentTopologyBuilder.java | 13 +-
storm-core/src/py/storm/DistributedRPC-remote | 0
.../py/storm/DistributedRPCInvocations-remote | 0
storm-core/src/py/storm/Nimbus-remote | 7 +
storm-core/src/py/storm/Nimbus.py | 226 +++++
storm-core/src/py/storm/ttypes.py | 80 ++
.../storm-core-version-info.properties | 24 +
storm-core/src/storm.thrift | 11 +
.../templates/topology-page-template.html | 8 +-
.../clj/backtype/storm/integration_test.clj | 10 +-
.../scheduler/multitenant_scheduler_test.clj | 2 +
.../test/clj/backtype/storm/supervisor_test.clj | 34 +-
.../storm/grouping/PartialKeyGroupingTest.java | 66 ++
storm-dist/binary/src/main/assembly/binary.xml | 7 -
71 files changed, 3647 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/STORM-UI-REST-API.md
----------------------------------------------------------------------
diff --cc STORM-UI-REST-API.md
index 4801a51,deee134..e677c11
--- a/STORM-UI-REST-API.md
+++ b/STORM-UI-REST-API.md
@@@ -250,8 -219,10 +250,11 @@@ Response fields
|bolts.errorWorkerLogLink| String | Link to the worker log that reported the exception |
|bolts.emitted| Long |Number of tuples emitted|
|antiForgeryToken| String | CSRF token|
+|replicationCount| Integer |Number of nimbus hosts on which this topology code is replicated|
+ Caution: users need to unescape the antiForgeryToken value before using this token to make POST calls(simple-json escapes forward slashes)
+ [ISSUE-8](https://code.google.com/p/json-simple/issues/detail?id=8)
+
Examples:
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/conf/defaults.yaml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index b35a8d2,af86bd0..a5543b9
--- a/pom.xml
+++ b/pom.xml
@@@ -209,9 -210,9 +210,8 @@@
<reply.version>0.3.0</reply.version>
<conjure.version>2.1.3</conjure.version>
<zookeeper.version>3.4.6</zookeeper.version>
- <conjure.version>2.1.3</conjure.version>
<clojure-data-codec.version>0.1.0</clojure-data-codec.version>
<clojure-contrib.version>1.2.0</clojure-contrib.version>
-
</properties>
<profiles>
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/clj/backtype/storm/cluster.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index 26bd591,0c7612b..30d123d
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@@ -24,13 -22,12 +24,14 @@@
(:use [backtype.storm.scheduler.DefaultScheduler])
(:import [backtype.storm.scheduler INimbus SupervisorDetails WorkerSlot TopologyDetails
Cluster Topologies SchedulerAssignment SchedulerAssignmentImpl DefaultScheduler ExecutorDetails])
+ (:use [backtype.storm bootstrap util zookeeper])
- (:import [backtype.storm.generated AuthorizationException])
+ (:import [backtype.storm.nimbus NimbusInfo])
- (:use [backtype.storm bootstrap util])
+ (:import [backtype.storm.generated AuthorizationException GetInfoOptions
+ NumErrorsChoice])
- (:use [backtype.storm bootstrap util])
(:use [backtype.storm.config :only [validate-configs-with-schemas]])
(:use [backtype.storm.daemon common])
(:import [org.apache.zookeeper data.ACL ZooDefs$Ids ZooDefs$Perms])
++ (:import [backtype.storm.utils VersionInfo])
(:gen-class
:methods [^{:static true} [launch [backtype.storm.scheduler.INimbus] void]]))
@@@ -1031,24 -991,9 +1039,24 @@@
(let [nimbus (nimbus-data conf inimbus)
principal-to-local (AuthUtils/GetPrincipalToLocalPlugin conf)]
(.prepare ^backtype.storm.nimbus.ITopologyValidator (:validator nimbus) conf)
+
+ ;add to nimbuses
+ (.add-nimbus-host! (:storm-cluster-state nimbus)
+ (.toHostPortString (:nimbus-host-port-info nimbus))
+ {
+ :host (.getHost (:nimbus-host-port-info nimbus))
+ :port (.getPort (:nimbus-host-port-info nimbus))
+ :start-time-secs (current-time-secs)
- :version (read-storm-version)
++ :version (str (VersionInfo/getVersion))
+ })
+
+ (.addToLeaderLockQueue (:leader-elector nimbus))
(cleanup-corrupt-topologies! nimbus)
- (doseq [storm-id (.active-storms (:storm-cluster-state nimbus))]
- (transition! nimbus storm-id :startup))
+ ;register call back for code-distributor
+ (.code-distributor (:storm-cluster-state nimbus) (fn [] (sync-code conf nimbus)))
+ (when (is-leader nimbus :throw-exception false)
+ (doseq [storm-id (.active-storms (:storm-cluster-state nimbus))]
+ (transition! nimbus storm-id :startup)))
(schedule-recurring (:timer nimbus)
0
(conf NIMBUS-MONITOR-FREQ-SECS)
@@@ -1332,11 -1259,11 +1340,11 @@@
topo-summ
))]
(ClusterSummary. supervisor-summaries
- nimbus-uptime
- topology-summaries)
+ topology-summaries
+ nimbuses)
))
- (^TopologyInfo getTopologyInfo [this ^String storm-id]
+ (^TopologyInfo getTopologyInfoWithOpts [this ^String storm-id ^GetInfoOptions options]
(let [storm-cluster-state (:storm-cluster-state nimbus)
topology-conf (try-read-storm-conf conf storm-id)
storm-name (topology-conf TOPOLOGY-NAME)
@@@ -1373,10 -1314,14 +1395,15 @@@
)]
(when-let [owner (:owner base)] (.set_owner topo-info owner))
(when-let [sched-status (.get @(:id->sched-status nimbus) storm-id)] (.set_sched_status topo-info sched-status))
+ (.set_replication_count topo-info (.getReplicationCount (:code-distributor nimbus) storm-id))
topo-info
))
-
+
+ (^TopologyInfo getTopologyInfo [this ^String storm-id]
+ (.getTopologyInfoWithOpts this
+ storm-id
+ (doto (GetInfoOptions.) (.set_num_err_choice NumErrorsChoice/ALL))))
+
Shutdownable
(shutdown [this]
(log-message "Shutting down master")
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 8fd22a6,b36a7a9..726da73
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -498,7 -504,8 +502,7 @@@
(map #(.get_num_executors ^TopologySummary %))
(reduce +))]
{"user" user
- "stormVersion" (read-storm-version)
+ "stormVersion" (str (VersionInfo/getVersion))
- "nimbusUptime" (pretty-uptime-sec (.get_nimbus_uptime_secs summ))
"supervisors" (count sups)
"slotsTotal" total-slots
"slotsUsed" used-slots
@@@ -654,10 -643,13 +658,13 @@@
"failed" (get-in stats [:failed k])})))
(defn topology-page [id window include-sys? user]
- (with-nimbus nimbus
+ (thrift/with-configured-nimbus-connection nimbus
(let [window (if window window ":all-time")
window-hint (window-hint window)
- summ (.getTopologyInfo ^Nimbus$Client nimbus id)
+ summ (->> (doto
+ (GetInfoOptions.)
+ (.set_num_err_choice NumErrorsChoice/ONE))
+ (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
topology (.getTopology ^Nimbus$Client nimbus id)
topology-conf (from-json (.getTopologyConf ^Nimbus$Client nimbus id))
spout-summs (filter (partial spout-summary? topology) (.get_executors summ))
@@@ -911,25 -898,36 +918,36 @@@
(let [user (.getUserName http-creds-handler servlet-request)]
(assert-authorized-user servlet-request "getTopology" (topology-config id))
(json-response (component-page id component (:window m) (check-include-sys? (:sys m)) user) (:callback m))))
+ (GET "/api/v1/token" [ & m]
+ (json-response (format "{\"antiForgeryToken\": \"%s\"}" *anti-forgery-token*) (:callback m) :serialize-fn identity))
(POST "/api/v1/topology/:id/activate" [:as {:keys [cookies servlet-request]} id]
- (with-nimbus nimbus
+ (thrift/with-configured-nimbus-connection nimbus
- (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (->> (doto
+ (GetInfoOptions.)
+ (.set_num_err_choice NumErrorsChoice/NONE))
+ (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
name (.get_name tplg)]
(assert-authorized-user servlet-request "activate" (topology-config id))
(.activate nimbus name)
(log-message "Activating topology '" name "'")))
- (resp/redirect (str "/api/v1/topology/" id)))
+ (resp/redirect (str "/api/v1/topology/" (url-encode id))))
(POST "/api/v1/topology/:id/deactivate" [:as {:keys [cookies servlet-request]} id]
- (with-nimbus nimbus
+ (thrift/with-configured-nimbus-connection nimbus
- (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (->> (doto
+ (GetInfoOptions.)
+ (.set_num_err_choice NumErrorsChoice/NONE))
+ (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
name (.get_name tplg)]
(assert-authorized-user servlet-request "deactivate" (topology-config id))
(.deactivate nimbus name)
(log-message "Deactivating topology '" name "'")))
(resp/redirect (str "/api/v1/topology/" (url-encode id))))
(POST "/api/v1/topology/:id/rebalance/:wait-time" [:as {:keys [cookies servlet-request]} id wait-time]
- (with-nimbus nimbus
+ (thrift/with-configured-nimbus-connection nimbus
- (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (->> (doto
+ (GetInfoOptions.)
+ (.set_num_err_choice NumErrorsChoice/NONE))
+ (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
name (.get_name tplg)
options (RebalanceOptions.)]
(assert-authorized-user servlet-request "rebalance" (topology-config id))
@@@ -938,8 -936,11 +956,11 @@@
(log-message "Rebalancing topology '" name "' with wait time: " wait-time " secs")))
(resp/redirect (str "/api/v1/topology/" (url-encode id))))
(POST "/api/v1/topology/:id/kill/:wait-time" [:as {:keys [cookies servlet-request]} id wait-time]
- (with-nimbus nimbus
+ (thrift/with-configured-nimbus-connection nimbus
- (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (->> (doto
+ (GetInfoOptions.)
+ (.set_num_err_choice NumErrorsChoice/NONE))
+ (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
name (.get_name tplg)
options (KillOptions.)]
(assert-authorized-user servlet-request "killTopology" (topology-config id))
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/py/storm/ttypes.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/storm.thrift
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------