You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/10/15 18:16:03 UTC
[50/50] git commit: Merge branch 'master' into security-upmerge
Merge branch 'master' into security-upmerge
Conflicts:
storm-core/src/clj/backtype/storm/daemon/supervisor.clj
storm-core/src/clj/backtype/storm/ui/core.clj
storm-core/src/ui/public/templates/topology-page-template.html
storm-core/test/clj/backtype/storm/supervisor_test.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ea0b8a2a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ea0b8a2a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ea0b8a2a
Branch: refs/heads/security
Commit: ea0b8a2a51332020a569ed7aeb3e0647477af763
Parents: 4788c7e cfe7e63
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Oct 15 11:05:10 2014 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Oct 15 11:05:10 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 16 +-
DEVELOPER.md | 26 +-
LICENSE | 20 +-
README.markdown | 3 +
bin/storm | 3 +-
examples/storm-starter/README.markdown | 9 +-
examples/storm-starter/pom.xml | 2 +-
external/storm-hbase/LICENSE | 202 +++++++++++
external/storm-hbase/README.md | 215 ++++++++++++
external/storm-hbase/pom.xml | 78 +++++
.../storm/hbase/bolt/AbstractHBaseBolt.java | 69 ++++
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 77 ++++
.../storm/hbase/bolt/HBaseLookupBolt.java | 83 +++++
.../storm/hbase/bolt/mapper/HBaseMapper.java | 48 +++
.../bolt/mapper/HBaseProjectionCriteria.java | 85 +++++
.../hbase/bolt/mapper/HBaseValueMapper.java | 43 +++
.../hbase/bolt/mapper/SimpleHBaseMapper.java | 90 +++++
.../apache/storm/hbase/common/ColumnList.java | 202 +++++++++++
.../apache/storm/hbase/common/HBaseClient.java | 135 +++++++
.../org/apache/storm/hbase/common/IColumn.java | 30 ++
.../org/apache/storm/hbase/common/ICounter.java | 29 ++
.../org/apache/storm/hbase/common/Utils.java | 69 ++++
.../storm/hbase/security/HBaseSecurityUtil.java | 52 +++
.../mapper/SimpleTridentHBaseMapper.java | 89 +++++
.../trident/mapper/TridentHBaseMapper.java | 31 ++
.../hbase/trident/state/HBaseMapState.java | 233 +++++++++++++
.../storm/hbase/trident/state/HBaseQuery.java | 40 +++
.../storm/hbase/trident/state/HBaseState.java | 164 +++++++++
.../hbase/trident/state/HBaseStateFactory.java | 40 +++
.../storm/hbase/trident/state/HBaseUpdater.java | 32 ++
.../storm/hbase/topology/LookupWordCount.java | 79 +++++
.../hbase/topology/PersistentWordCount.java | 81 +++++
.../storm/hbase/topology/TotalWordCounter.java | 70 ++++
.../storm/hbase/topology/WordCountClient.java | 57 +++
.../hbase/topology/WordCountValueMapper.java | 70 ++++
.../storm/hbase/topology/WordCounter.java | 59 ++++
.../apache/storm/hbase/topology/WordSpout.java | 88 +++++
.../storm/hbase/trident/PrintFunction.java | 40 +++
.../storm/hbase/trident/WordCountTrident.java | 104 ++++++
external/storm-hdfs/pom.xml | 2 +-
external/storm-kafka/pom.xml | 2 +-
.../jvm/storm/kafka/DynamicBrokersReader.java | 2 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 1 +
.../src/test/storm/kafka/TridentKafkaTest.java | 9 +-
pom.xml | 27 +-
.../maven-shade-clojure-transformer/pom.xml | 2 +-
storm-core/pom.xml | 6 +-
.../clj/backtype/storm/daemon/supervisor.clj | 4 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 51 +--
storm-core/src/dev/resources/storm.py | 248 ++++++++++++-
storm-core/src/dev/resources/storm.rb | 228 +++++++++++-
.../src/jvm/backtype/storm/tuple/ITuple.java | 136 ++++++++
.../src/jvm/backtype/storm/tuple/Tuple.java | 113 +-----
.../storm/trident/testing/MockTridentTuple.java | 166 ---------
.../jvm/storm/trident/tuple/TridentTuple.java | 46 +--
.../storm/trident/tuple/TridentTupleView.java | 35 +-
.../jvm/storm/trident/tuple/ValuePointer.java | 1 -
storm-core/src/multilang/js/storm.js | 349 +++++++++++++++++++
storm-core/src/ui/public/component.html | 8 +-
storm-core/src/ui/public/js/purl.js | 267 --------------
storm-core/src/ui/public/js/script.js | 3 +-
storm-core/src/ui/public/js/url.min.js | 1 +
storm-core/src/ui/public/js/visualization.js | 2 +-
.../templates/component-page-template.html | 6 +-
.../public/templates/index-page-template.html | 2 +-
.../templates/topology-page-template.html | 6 +-
storm-core/src/ui/public/topology.html | 12 +-
.../test/clj/backtype/storm/supervisor_test.clj | 12 +-
.../test/clj/storm/trident/tuple_test.clj | 28 +-
storm-dist/binary/LICENSE | 10 +-
storm-dist/binary/pom.xml | 2 +-
storm-dist/binary/src/main/assembly/binary.xml | 14 +
storm-dist/source/pom.xml | 2 +-
73 files changed, 3949 insertions(+), 717 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/README.markdown
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/bin/storm
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 1fe5b2e,cce89bc..c580bc8
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -605,9 -501,8 +605,9 @@@
(defmethod launch-worker
:distributed [supervisor storm-id port worker-id]
(let [conf (:conf supervisor)
+ run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
storm-home (System/getProperty "storm.home")
- storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home "/logs"))
+ storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home file-path-separator "logs"))
stormroot (supervisor-stormdist-root conf storm-id)
jlp (jlp stormroot conf)
stormjar (supervisor-stormjar-path stormroot)
@@@ -638,7 -529,7 +638,7 @@@
(str "-Dlogfile.name=" logfilename)
(str "-Dstorm.home=" storm-home)
(str "-Dstorm.log.dir=" storm-log-dir)
- (str "-Dlogback.configurationFile=" storm-home "/logback/worker.xml")
- (str "-Dlogback.configurationFile=" storm-home file-path-separator "logback" file-path-separator "cluster.xml")
++ (str "-Dlogback.configurationFile=" storm-home file-path-separator "logback" file-path-separator "worker.xml")
(str "-Dstorm.id=" storm-id)
(str "-Dworker.id=" worker-id)
(str "-Dworker.port=" port)
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 600c96a,969b514..e980a55
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -541,8 -519,9 +542,10 @@@
([summs]
{"topologies"
(for [^TopologySummary t summs]
- {"id" (.get_id t)
+ {
+ "id" (.get_id t)
+ "encodedId" (url-encode (.get_id t))
+ "owner" (.get_owner t)
"name" (.get_name t)
"status" (.get_status t)
"uptime" (pretty-uptime-sec (.get_uptime_secs t))
@@@ -573,8 -551,9 +576,9 @@@
stats-seq include-sys?))
last-error (most-recent-error (get errors id))
error-host (get-error-host last-error)
- error-port (get-error-port last-error error-host top-id) ]]
+ error-port (get-error-port last-error error-host top-id)]]
{"spoutId" id
+ "encodedSpoutId" (url-encode id)
"executors" (count summs)
"tasks" (sum-tasks summs)
"emitted" (get-in stats [:emitted window])
@@@ -596,8 -575,9 +600,9 @@@
stats-seq include-sys?))
last-error (most-recent-error (get errors id))
error-host (get-error-host last-error)
- error-port (get-error-port last-error error-host top-id) ]]
+ error-port (get-error-port last-error error-host top-id)]]
{"boltId" id
+ "encodedBoltId" (url-encode id)
"executors" (count summs)
"tasks" (sum-tasks summs)
"emitted" (get-in stats [:emitted window])
@@@ -619,7 -599,7 +624,8 @@@
workers (set (for [^ExecutorSummary e executors]
[(.get_host e) (.get_port e)]))]
{"id" (.get_id summ)
+ "encodedId" (url-encode (.get_id summ))
+ "owner" (.get_owner summ)
"name" (.get_name summ)
"status" (.get_status summ)
"uptime" (pretty-uptime-sec (.get_uptime_secs summ))
@@@ -832,8 -811,8 +841,9 @@@
(= type :bolt) (bolt-stats window summ component summs include-sys?))
errors (component-errors (get (.get_errors summ) component) topology-id)]
(merge
- { "id" component
+ {"user" user
+ "id" component
+ "encodedId" (url-encode component)
"name" (.get_name summ)
"executors" (count summs)
"tasks" (sum-tasks summs)
@@@ -869,67 -843,47 +880,59 @@@
(GET "/api/v1/cluster/configuration" [& m]
(json-response (cluster-configuration)
(:callback m) :serialize-fn identity))
- (GET "/api/v1/cluster/summary" [& m]
- (json-response (cluster-summary) (:callback m)))
- (GET "/api/v1/supervisor/summary" [& m]
+ (GET "/api/v1/cluster/summary" [:as {:keys [cookies servlet-request]} & m]
+ (let [user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getClusterInfo")
+ (json-response (cluster-summary user) (:callback m))))
+ (GET "/api/v1/supervisor/summary" [:as {:keys [cookies servlet-request]} & m]
+ (assert-authorized-user servlet-request "getClusterInfo")
(json-response (supervisor-summary) (:callback m)))
- (GET "/api/v1/topology/summary" [& m]
+ (GET "/api/v1/topology/summary" [:as {:keys [cookies servlet-request]} & m]
+ (assert-authorized-user servlet-request "getClusterInfo")
(json-response (all-topologies-summary) (:callback m)))
- (GET "/api/v1/topology/:id" [id & m]
- (json-response (topology-page id (:window m) (check-include-sys? (:sys m))) (:callback m)))
+ (GET "/api/v1/topology/:id" [:as {:keys [cookies servlet-request]} id & m]
- (let [id (url-decode id)
- user (.getUserName http-creds-handler servlet-request)]
++ (let [user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
+ (json-response (topology-page id (:window m) (check-include-sys? (:sys m)) user) (:callback m))))
(GET "/api/v1/topology/:id/visualization" [:as {:keys [cookies servlet-request]} id & m]
- (let [id (url-decode id)]
- (assert-authorized-user servlet-request "getTopology" (topology-config id))
- (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))) (:callback m))))
- (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))) (:callback m)))
- (GET "/api/v1/topology/:id/component/:component" [id component & m]
- (json-response (component-page id component (:window m) (check-include-sys? (:sys m))) (:callback m)))
- (POST "/api/v1/topology/:id/activate" [id]
++ (assert-authorized-user servlet-request "getTopology" (topology-config id))
++ (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m))) (:callback m)))
+ (GET "/api/v1/topology/:id/component/:component" [:as {:keys [cookies servlet-request]} id component & m]
- (let [id (url-decode id)
- component (url-decode component)
- user (.getUserName http-creds-handler servlet-request)]
++ (let [user (.getUserName http-creds-handler servlet-request)]
+ (assert-authorized-user servlet-request "getTopology" (topology-config id))
+ (json-response (component-page id component (:window m) (check-include-sys? (:sys m)) user) (:callback m))))
+ (POST "/api/v1/topology/:id/activate" [:as {:keys [cookies servlet-request]} id]
(with-nimbus nimbus
- (let [id (url-decode id)
- tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)]
+ (assert-authorized-user servlet-request "activate" (topology-config id))
(.activate nimbus name)
(log-message "Activating topology '" name "'")))
- (resp/redirect (str "/api/v1/topology/" (url-encode id))))
-
- (POST "/api/v1/topology/:id/deactivate" [id]
+ (resp/redirect (str "/api/v1/topology/" id)))
+ (POST "/api/v1/topology/:id/deactivate" [:as {:keys [cookies servlet-request]} id]
(with-nimbus nimbus
- (let [id (url-decode id)
- tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)]
+ (assert-authorized-user servlet-request "deactivate" (topology-config id))
(.deactivate nimbus name)
(log-message "Deactivating topology '" name "'")))
- (resp/redirect (str "/api/v1/topology/" id)))
+ (resp/redirect (str "/api/v1/topology/" (url-encode id))))
- (POST "/api/v1/topology/:id/rebalance/:wait-time" [id wait-time]
+ (POST "/api/v1/topology/:id/rebalance/:wait-time" [:as {:keys [cookies servlet-request]} id wait-time]
(with-nimbus nimbus
- (let [id (url-decode id)
- tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)
options (RebalanceOptions.)]
+ (assert-authorized-user servlet-request "rebalance" (topology-config id))
(.set_wait_secs options (Integer/parseInt wait-time))
(.rebalance nimbus name options)
(log-message "Rebalancing topology '" name "' with wait time: " wait-time " secs")))
- (resp/redirect (str "/api/v1/topology/" id)))
+ (resp/redirect (str "/api/v1/topology/" (url-encode id))))
- (POST "/api/v1/topology/:id/kill/:wait-time" [id wait-time]
+ (POST "/api/v1/topology/:id/kill/:wait-time" [:as {:keys [cookies servlet-request]} id wait-time]
(with-nimbus nimbus
- (let [id (url-decode id)
- tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+ (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
name (.get_name tplg)
options (KillOptions.)]
+ (assert-authorized-user servlet-request "killTopology" (topology-config id))
(.set_wait_secs options (Integer/parseInt wait-time))
(.killTopologyWithOpts nimbus name options)
(log-message "Killing topology '" name "' with wait time: " wait-time " secs")))
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/index-page-template.html
index 800b579,e3aab5e..9c48e83
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@@ -128,9 -118,8 +128,9 @@@
<tbody>
{{#topologies}}
<tr>
- <td><a href="/topology.html?id={{id}}">{{name}}</a></td>
+ <td><a href="/topology.html?id={{encodedId}}">{{name}}</a></td>
<td>{{id}}</td>
+ <td>{{owner}}</td>
<td>{{status}}</td>
<td>{{uptime}}</td>
<td>{{workersTotal}}</td>
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index 4261b4d,ba74d88..ebbf060
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -259,10 -253,10 +259,10 @@@
opts
topo-opts
["-Djava.library.path="
- (str "-Dlogfile.name=worker-" mock-port ".log")
+ (str "-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log")
"-Dstorm.home="
- "-Dstorm.log.dir=/logs"
- "-Dlogback.configurationFile=/logback/worker.xml"
+ (str "-Dstorm.log.dir=" file-path-separator "logs")
- (str "-Dlogback.configurationFile=" file-path-separator "logback" file-path-separator "cluster.xml")
++ (str "-Dlogback.configurationFile=" file-path-separator "logback" file-path-separator "worker.xml")
(str "-Dstorm.id=" mock-storm-id)
(str "-Dworker.id=" mock-worker-id)
(str "-Dworker.port=" mock-port)
@@@ -322,10 -312,8 +322,10 @@@
(stubbing [read-supervisor-storm-conf {TOPOLOGY-CLASSPATH topo-cp}
supervisor-stormdist-root nil
supervisor/jlp nil
+ set-worker-user! nil
+ supervisor/write-log-metadata! nil
launch-process nil
- current-classpath "/base"]
+ current-classpath (str file-path-separator "base")]
(supervisor/launch-worker mock-supervisor
mock-storm-id
mock-port
@@@ -342,9 -329,7 +342,9 @@@
supervisor-stormdist-root nil
supervisor/jlp nil
launch-process nil
+ set-worker-user! nil
+ supervisor/write-log-metadata! nil
- current-classpath "/base"]
+ current-classpath (str file-path-separator "base")]
(supervisor/launch-worker mock-supervisor
mock-storm-id
mock-port
http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------