You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/01/06 23:02:36 UTC
[1/7] storm git commit: Found a bug and fixed some tests to validate
the bug is fixed.
Repository: storm
Updated Branches:
refs/heads/master 5314b88a4 -> c092cc5cd
Found a bug and fixed some tests to validate the bug is fixed.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/68f4bcc3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/68f4bcc3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/68f4bcc3
Branch: refs/heads/master
Commit: 68f4bcc383e23f103d8f62af97df69ad4a4962d0
Parents: f3e3aa8
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Nov 18 15:17:21 2014 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Nov 20 09:41:42 2014 -0600
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/logviewer.clj | 2 +-
.../test/clj/backtype/storm/logviewer_test.clj | 39 ++++++++++++++------
.../backtype/storm/security/auth/auth_test.clj | 8 ++--
3 files changed, 33 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/68f4bcc3/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
index 978d975..44c49df 100644
--- a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
@@ -192,7 +192,7 @@
(defn authorized-log-user? [user fname conf]
(if (or (blank? user) (blank? fname))
nil
- (let [groups (user-groups fname)
+ (let [groups (user-groups user)
[user-wl group-wl] (get-log-user-group-whitelist fname)
logs-users (concat (conf LOGS-USERS)
(conf NIMBUS-ADMINS)
http://git-wip-us.apache.org/repos/asf/storm/blob/68f4bcc3/storm-core/test/clj/backtype/storm/logviewer_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/logviewer_test.clj b/storm-core/test/clj/backtype/storm/logviewer_test.clj
index 93f22e6..0600903 100644
--- a/storm-core/test/clj/backtype/storm/logviewer_test.clj
+++ b/storm-core/test/clj/backtype/storm/logviewer_test.clj
@@ -165,28 +165,45 @@
(deftest test-authorized-log-user
(testing "allow cluster admin"
(let [conf {NIMBUS-ADMINS ["alice"]}]
- (stubbing [logviewer/get-log-user-group-whitelist [[] []]]
- (is (logviewer/authorized-log-user? "alice" "non-blank-fname" conf)))))
+ (stubbing [logviewer/get-log-user-group-whitelist [[] []]
+ logviewer/user-groups []]
+ (is (logviewer/authorized-log-user? "alice" "non-blank-fname" conf))
+ (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
+ (verify-first-call-args-for logviewer/user-groups "alice"))))
(testing "ignore any cluster-set topology.users"
(let [conf {TOPOLOGY-USERS ["alice"]}]
- (stubbing [logviewer/get-log-user-group-whitelist [[] []]]
- (is (not (logviewer/authorized-log-user? "alice" "non-blank-fname" conf))))))
+ (stubbing [logviewer/get-log-user-group-whitelist [[] []]
+ logviewer/user-groups []]
+ (is (not (logviewer/authorized-log-user? "alice" "non-blank-fname" conf)))
+ (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
+ (verify-first-call-args-for logviewer/user-groups "alice"))))
(testing "allow cluster logs user"
(let [conf {LOGS-USERS ["alice"]}]
- (stubbing [logviewer/get-log-user-group-whitelist [[] []]]
- (is (logviewer/authorized-log-user? "alice" "non-blank-fname" conf)))))
+ (stubbing [logviewer/get-log-user-group-whitelist [[] []]
+ logviewer/user-groups []]
+ (is (logviewer/authorized-log-user? "alice" "non-blank-fname" conf))
+ (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
+ (verify-first-call-args-for logviewer/user-groups "alice"))))
(testing "allow whitelisted topology user"
- (stubbing [logviewer/get-log-user-group-whitelist [["alice"] []]]
- (is (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))))
+ (stubbing [logviewer/get-log-user-group-whitelist [["alice"] []]
+ logviewer/user-groups []]
+ (is (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))
+ (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
+ (verify-first-call-args-for logviewer/user-groups "alice")))
(testing "allow whitelisted topology group"
(stubbing [logviewer/get-log-user-group-whitelist [[] ["alice-group"]]
logviewer/user-groups ["alice-group"]]
- (is (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))))
+ (is (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))
+ (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
+ (verify-first-call-args-for logviewer/user-groups "alice")))
(testing "disallow user not in nimbus admin, topo user, logs user, or whitelist"
- (stubbing [logviewer/get-log-user-group-whitelist [[] []]]
- (is (not (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))))))
+ (stubbing [logviewer/get-log-user-group-whitelist [[] []]
+ logviewer/user-groups []]
+ (is (not (logviewer/authorized-log-user? "alice" "non-blank-fname" {})))
+ (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
+ (verify-first-call-args-for logviewer/user-groups "alice"))))
http://git-wip-us.apache.org/repos/asf/storm/blob/68f4bcc3/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj b/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
index a01cac3..6563730 100644
--- a/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
+++ b/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
@@ -279,13 +279,13 @@
))
(deftest shell-based-groups-mapping-test
- (let [cluster-conf (merge (read-storm-config))
+ (let [cluster-conf (read-storm-config)
groups (ShellBasedGroupsMapping. )
user-name (System/getProperty "user.name")]
(.prepare groups cluster-conf)
- (>= 0 (.size (.getGroups groups user-name)))
- (= 0 (.size (.getGroups groups "userDoesNotExist")))
- (= 0 (.size (.getGroups groups nil)))))
+ (is (<= 0 (.size (.getGroups groups user-name))))
+ (is (= 0 (.size (.getGroups groups "userDoesNotExist"))))
+ (is (= 0 (.size (.getGroups groups nil))))))
(deftest simple-acl-same-user-auth-test
(let [cluster-conf (merge (read-storm-config)
[6/7] storm git commit: Merge branch 'STORM-410' of
https://github.com/revans2/incubator-storm into STORM-410
Posted by bo...@apache.org.
Merge branch 'STORM-410' of https://github.com/revans2/incubator-storm into STORM-410
STORM-410: Add groups support to log-viewer
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bab1f566
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bab1f566
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bab1f566
Branch: refs/heads/master
Commit: bab1f566cdc8059d4c8bddc3fe30ba5ed498e7df
Parents: 5314b88 231ac22
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Jan 6 15:54:12 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Jan 6 15:54:12 2015 -0600
----------------------------------------------------------------------
SECURITY.md | 4 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 27 ++++++++---
.../clj/backtype/storm/daemon/supervisor.clj | 4 ++
storm-core/src/jvm/backtype/storm/Config.java | 12 ++---
.../test/clj/backtype/storm/logviewer_test.clj | 47 +++++++++++++++-----
.../backtype/storm/security/auth/auth_test.clj | 8 ++--
.../test/clj/backtype/storm/supervisor_test.clj | 6 ++-
7 files changed, 76 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bab1f566/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bab1f566/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bab1f566/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
[7/7] storm git commit: Added STORM-410 to Changelog
Posted by bo...@apache.org.
Added STORM-410 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c092cc5c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c092cc5c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c092cc5c
Branch: refs/heads/master
Commit: c092cc5cdfc8dfefaa3900719af619122ac255aa
Parents: bab1f56
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Jan 6 15:54:50 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Jan 6 15:54:50 2015 -0600
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c092cc5c/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 529361e..1f55539 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,7 @@
* STORM-610: Check the return value of fts_close()
* STORM-442: multilang ShellBolt/ShellSpout die() can be hang when Exception happened
* STORM-599: Use use nimbus's cached heartbeats rather than fetching again from ZK
+ * STORM-410: Add groups support to log-viewer
## 0.9.3-rc2
* STORM-558: change "swap!" to "reset!" to fix assignment-versions in supervisor
[4/7] storm git commit: Merge branch 'apache-master' into STORM-410
Posted by bo...@apache.org.
Merge branch 'apache-master' into STORM-410
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cdc699ef
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cdc699ef
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cdc699ef
Branch: refs/heads/master
Commit: cdc699efd13c4ddc45565826ae51bf60ee3e1c53
Parents: 618e485 da02d8c
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Dec 22 14:56:37 2014 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Dec 22 14:56:37 2014 -0600
----------------------------------------------------------------------
.gitignore | 1 +
CHANGELOG.md | 12 +-
README.markdown | 2 +
STORM-UI-REST-API.md | 186 +-
bin/storm | 59 +-
conf/defaults.yaml | 1 +
doap_Storm.rdf | 57 +
docs/README.md | 38 +
docs/_config.yml | 11 +
docs/_includes/footer.html | 16 +
docs/_includes/head.html | 31 +
docs/_includes/header.html | 25 +
docs/_layouts/about.html | 40 +
docs/_layouts/default.html | 19 +
docs/_layouts/documentation.html | 16 +
docs/_layouts/page.html | 14 +
docs/_layouts/post.html | 15 +
docs/_posts/2012-08-02-storm080-released.md | 120 +
docs/_posts/2012-09-06-storm081-released.md | 47 +
docs/_posts/2013-01-11-storm082-released.md | 82 +
docs/_posts/2013-12-08-storm090-released.md | 127 +
docs/_posts/2014-04-10-storm-logo-contest.md | 66 +
docs/_posts/2014-04-17-logo-pforrest.md | 10 +
docs/_posts/2014-04-17-logo-squinones.md | 9 +
docs/_posts/2014-04-19-logo-ssuleman.md | 8 +
docs/_posts/2014-04-21-logo-rmarshall.md | 12 +
docs/_posts/2014-04-22-logo-zsayari.md | 9 +
docs/_posts/2014-04-23-logo-abartos.md | 15 +
docs/_posts/2014-04-27-logo-cboustead.md | 12 +
docs/_posts/2014-04-27-logo-sasili.md | 10 +
docs/_posts/2014-04-29-logo-jlee1.md | 10 +
docs/_posts/2014-04-29-logo-jlee2.md | 10 +
docs/_posts/2014-04-29-logo-jlee3.md | 10 +
docs/_posts/2014-05-27-round1-results.md | 38 +
docs/_posts/2014-06-17-contest-results.md | 24 +
docs/_posts/2014-06-25-storm092-released.md | 137 +
.../2014-10-20-storm093-release-candidate.md | 11 +
docs/_posts/2014-11-25-storm093-released.md | 164 +
docs/_sass/_syntax-highlighting.scss | 70 +
docs/about.md | 7 +
docs/about/deployment.md | 9 +
docs/about/fault-tolerant.md | 9 +
docs/about/free-and-open-source.md | 15 +
docs/about/guarantees-data-processing.md | 10 +
docs/about/integrates.md | 13 +
docs/about/multi-language.md | 9 +
docs/about/scalable.md | 10 +
docs/about/simple-api.md | 15 +
docs/assets/css/bootstrap-theme.css | 470 ++
docs/assets/css/bootstrap-theme.css.map | 1 +
docs/assets/css/bootstrap-theme.min.css | 5 +
docs/assets/css/bootstrap.css | 6332 ++++++++++++++++++
docs/assets/css/bootstrap.css.map | 1 +
docs/assets/css/bootstrap.min.css | 5 +
docs/assets/css/theme.css | 18 +
docs/assets/favicon.ico | Bin 0 -> 1150 bytes
.../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20335 bytes
.../fonts/glyphicons-halflings-regular.svg | 229 +
.../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 41280 bytes
.../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23320 bytes
docs/assets/js/bootstrap.js | 2320 +++++++
docs/assets/js/bootstrap.min.js | 7 +
docs/assets/js/npm.js | 13 +
docs/css/main.scss | 47 +
docs/doc-index.html | 11 +
.../Acking-framework-implementation.md | 38 +
docs/documentation/Clojure-DSL.md | 266 +
docs/documentation/Command-line-client.md | 102 +
docs/documentation/Common-patterns.md | 88 +
docs/documentation/Concepts.md | 117 +
docs/documentation/Configuration.md | 31 +
docs/documentation/Contributing-to-Storm.md | 33 +
.../Creating-a-new-Storm-project.md | 27 +
.../DSLs-and-multilang-adapters.md | 11 +
...Defining-a-non-jvm-language-dsl-for-storm.md | 38 +
docs/documentation/Distributed-RPC.md | 199 +
docs/documentation/Documentation.md | 52 +
docs/documentation/FAQ.md | 123 +
docs/documentation/Fault-tolerance.md | 30 +
.../Guaranteeing-message-processing.md | 181 +
docs/documentation/Home.md | 69 +
docs/documentation/Hooks.md | 9 +
docs/documentation/Implementation-docs.md | 20 +
.../Installing-native-dependencies.md | 38 +
docs/documentation/Kestrel-and-Storm.md | 200 +
docs/documentation/Lifecycle-of-a-topology.md | 82 +
docs/documentation/Local-mode.md | 29 +
docs/documentation/Maven.md | 56 +
.../Message-passing-implementation.md | 30 +
docs/documentation/Metrics.md | 36 +
docs/documentation/Multilang-protocol.md | 223 +
docs/documentation/Powered-By.md | 925 +++
docs/documentation/Project-ideas.md | 6 +
docs/documentation/Rationale.md | 33 +
...unning-topologies-on-a-production-cluster.md | 77 +
.../Serialization-(prior-to-0.6.0).md | 52 +
docs/documentation/Serialization.md | 62 +
docs/documentation/Serializers.md | 4 +
.../documentation/Setting-up-a-Storm-cluster.md | 85 +
.../Setting-up-a-Storm-project-in-Eclipse.md | 1 +
.../Setting-up-development-environment.md | 41 +
docs/documentation/Spout-implementations.md | 10 +
...guage-protocol-(versions-0.7.0-and-below).md | 124 +
docs/documentation/Structure-of-the-codebase.md | 142 +
.../Support-for-non-java-languages.md | 9 +
docs/documentation/Transactional-topologies.md | 361 +
docs/documentation/Trident-API-Overview.md | 312 +
docs/documentation/Trident-spouts.md | 44 +
docs/documentation/Trident-state.md | 331 +
docs/documentation/Trident-tutorial.md | 254 +
docs/documentation/Troubleshooting.md | 145 +
docs/documentation/Tutorial.md | 312 +
...nding-the-parallelism-of-a-Storm-topology.md | 123 +
.../Using-non-JVM-languages-with-Storm.md | 52 +
docs/documentation/images/ack_tree.png | Bin 0 -> 31463 bytes
docs/documentation/images/batched-stream.png | Bin 0 -> 66336 bytes
docs/documentation/images/drpc-workflow.png | Bin 0 -> 66199 bytes
.../images/eclipse-project-properties.png | Bin 0 -> 80810 bytes
.../images/example-of-a-running-topology.png | Bin 0 -> 81430 bytes
docs/documentation/images/grouping.png | Bin 0 -> 39701 bytes
.../images/ld-library-path-eclipse-linux.png | Bin 0 -> 114597 bytes
...onships-worker-processes-executors-tasks.png | Bin 0 -> 54804 bytes
docs/documentation/images/spout-vs-state.png | Bin 0 -> 24804 bytes
docs/documentation/images/storm-cluster.png | Bin 0 -> 34604 bytes
docs/documentation/images/topology-tasks.png | Bin 0 -> 45960 bytes
docs/documentation/images/topology.png | Bin 0 -> 23147 bytes
.../images/transactional-batches.png | Bin 0 -> 23293 bytes
.../images/transactional-commit-flow.png | Bin 0 -> 17725 bytes
.../images/transactional-design-2.png | Bin 0 -> 13537 bytes
.../images/transactional-spout-structure.png | Bin 0 -> 25067 bytes
docs/documentation/images/trident-to-storm1.png | Bin 0 -> 67173 bytes
docs/documentation/images/trident-to-storm2.png | Bin 0 -> 68943 bytes
docs/documentation/images/tuple-dag.png | Bin 0 -> 18849 bytes
docs/documentation/images/tuple_tree.png | Bin 0 -> 58186 bytes
docs/downloads.html | 155 +
docs/feed.xml | 30 +
docs/images/bullet.gif | Bin 0 -> 82 bytes
docs/images/download.png | Bin 0 -> 16272 bytes
docs/images/incubator-logo.png | Bin 0 -> 11651 bytes
.../logocontest/abartos/stationery_mockup.jpg | Bin 0 -> 146498 bytes
docs/images/logocontest/abartos/storm_logo.png | Bin 0 -> 153974 bytes
docs/images/logocontest/abartos/storm_logo2.png | Bin 0 -> 115425 bytes
docs/images/logocontest/abartos/storm_logo3.png | Bin 0 -> 94950 bytes
.../images/logocontest/cboustead/storm_logo.png | Bin 0 -> 67149 bytes
.../logocontest/cboustead/storm_logo1.png | Bin 0 -> 16327 bytes
docs/images/logocontest/jlee1/storm_logo.jpg | Bin 0 -> 189382 bytes
docs/images/logocontest/jlee2/storm_logo.jpg | Bin 0 -> 155666 bytes
docs/images/logocontest/jlee3/storm_logo.jpg | Bin 0 -> 158134 bytes
docs/images/logocontest/pforrest/storm1.png | Bin 0 -> 84569 bytes
.../pforrest/storm_logo_composite.png | Bin 0 -> 139223 bytes
.../rmarshall/StormLogo_Horizontal.png | Bin 0 -> 16481 bytes
.../rmarshall/StormLogo_Horizontal_NoColour.png | Bin 0 -> 14358 bytes
.../logocontest/rmarshall/StormLogo_Square.png | Bin 0 -> 14392 bytes
docs/images/logocontest/sasili/storm_logo.png | Bin 0 -> 92196 bytes
.../images/logocontest/squinones/storm_logo.png | Bin 0 -> 203263 bytes
.../logocontest/squinones/storm_logo1.png | Bin 0 -> 53325 bytes
docs/images/logocontest/ssuleman/storm_logo.png | Bin 0 -> 95509 bytes
docs/images/logocontest/storm_logo_winner.png | Bin 0 -> 34490 bytes
docs/images/logocontest/zsayari/storm_logo.png | Bin 0 -> 120794 bytes
docs/images/logos/8digits.png | Bin 0 -> 19557 bytes
docs/images/logos/Yahoo_Japan_logo.png | Bin 0 -> 3707 bytes
docs/images/logos/aeris.png | Bin 0 -> 6268 bytes
docs/images/logos/alibaba.jpg | Bin 0 -> 43703 bytes
docs/images/logos/baidu.jpeg | Bin 0 -> 3413 bytes
docs/images/logos/cerner.gif | Bin 0 -> 2591 bytes
docs/images/logos/flipboard.jpeg | Bin 0 -> 2909 bytes
docs/images/logos/fullcontact.png | Bin 0 -> 24567 bytes
docs/images/logos/groupon.jpg | Bin 0 -> 41413 bytes
docs/images/logos/holidaycheck.png | Bin 0 -> 3129 bytes
docs/images/logos/idexx.gif | Bin 0 -> 38689 bytes
docs/images/logos/mercadolibre.png | Bin 0 -> 73388 bytes
docs/images/logos/navisite.jpg | Bin 0 -> 9358 bytes
docs/images/logos/ooyala.gif | Bin 0 -> 7830 bytes
docs/images/logos/parc.png | Bin 0 -> 7256 bytes
docs/images/logos/quicklizard.png | Bin 0 -> 5667 bytes
docs/images/logos/rocketfuel.png | Bin 0 -> 9719 bytes
docs/images/logos/rubicon.png | Bin 0 -> 10663 bytes
docs/images/logos/spiderio.png | Bin 0 -> 46790 bytes
docs/images/logos/spotify.jpeg | Bin 0 -> 3282 bytes
docs/images/logos/taobao.gif | Bin 0 -> 3262 bytes
docs/images/logos/twitter.png | Bin 0 -> 4392 bytes
docs/images/logos/weatherchannel.gif | Bin 0 -> 3425 bytes
docs/images/logos/webmd.jpg | Bin 0 -> 6193 bytes
docs/images/logos/yelp.png | Bin 0 -> 98431 bytes
docs/images/mailinglist.png | Bin 0 -> 4245 bytes
docs/images/storm_header.png | Bin 0 -> 17291 bytes
docs/images/storm_logo_tagline_color copy.png | Bin 0 -> 67928 bytes
docs/images/storm_logo_tagline_color.png | Bin 0 -> 33568 bytes
docs/images/top_bg.gif | Bin 0 -> 113 bytes
docs/images/topology.png | Bin 0 -> 59837 bytes
docs/images/ui_topology_viz.png | Bin 0 -> 112831 bytes
docs/index.html | 104 +
docs/news.html | 12 +
.../multilang/resources/asyncSplitsentence.js | 18 +
.../multilang/resources/randomsentence.js | 18 +
.../multilang/resources/splitsentence.js | 18 +
.../storm-starter/multilang/resources/storm.js | 18 +
external/storm-hbase/pom.xml | 1 -
.../trident/mapper/TridentHBaseMapper.java | 18 +
.../hbase/trident/state/HBaseMapState.java | 17 +
.../hdfs/bolt/format/DefaultSequenceFormat.java | 17 +
.../storm/hdfs/bolt/format/SequenceFormat.java | 17 +
.../hdfs/bolt/rotation/TimedRotationPolicy.java | 17 +
.../hdfs/common/rotation/MoveFileAction.java | 17 +
.../hdfs/common/rotation/RotationAction.java | 17 +
.../apache/storm/hdfs/trident/HdfsState.java | 17 +
.../storm/hdfs/trident/HdfsStateFactory.java | 17 +
.../apache/storm/hdfs/trident/HdfsUpdater.java | 17 +
.../trident/format/DefaultSequenceFormat.java | 17 +
.../hdfs/trident/format/SequenceFormat.java | 17 +
.../trident/rotation/TimedRotationPolicy.java | 17 +
.../storm/hdfs/trident/FixedBatchSpout.java | 17 +
.../storm/hdfs/trident/TridentFileTopology.java | 17 +
.../hdfs/trident/TridentSequenceTopology.java | 17 +
external/storm-kafka/README.md | 6 +-
.../jvm/storm/kafka/UpdateOffsetException.java | 17 +
.../FieldNameBasedTupleToKafkaMapper.java | 2 +-
logback/cluster.xml | 2 +-
pom.xml | 11 +-
.../src/clj/backtype/storm/daemon/executor.clj | 6 +-
.../src/clj/backtype/storm/messaging/loader.clj | 3 +-
storm-core/src/clj/backtype/storm/testing4j.clj | 23 +-
storm-core/src/dev/resources/storm.js | 18 +
storm-core/src/dev/resources/tester_bolt.js | 17 +
storm-core/src/dev/resources/tester_spout.js | 17 +
storm-core/src/jvm/backtype/storm/Config.java | 10 +
.../auth/authorizer/SimpleACLAuthorizer.java | 2 +-
.../storm/testing/CompleteTopologyParam.java | 22 +-
.../storm/testing/PythonShellMetricsBolt.java | 17 +
.../storm/testing/PythonShellMetricsSpout.java | 17 +
.../storm/ui/InvalidRequestException.java | 17 +
storm-core/src/multilang/js/storm.js | 19 +-
storm-core/src/ui/public/component.html | 2 +-
storm-core/src/ui/public/favicon.ico | Bin 0 -> 18280 bytes
234 files changed, 17489 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/cdc699ef/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
[2/7] storm git commit: [STORM-410] Add groups support to log-viewer
Posted by bo...@apache.org.
[STORM-410] Add groups support to log-viewer
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f3e3aa82
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f3e3aa82
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f3e3aa82
Branch: refs/heads/master
Commit: f3e3aa82e567d4c9c8a70ceb854cfe55c57f4639
Parents: f8bce22
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Nov 18 09:30:54 2014 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Nov 20 09:41:42 2014 -0600
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/logviewer.clj | 27 +++++++++++++++-----
.../clj/backtype/storm/daemon/supervisor.clj | 4 +++
storm-core/src/jvm/backtype/storm/Config.java | 12 ++++-----
.../test/clj/backtype/storm/logviewer_test.clj | 15 +++++++----
.../test/clj/backtype/storm/supervisor_test.clj | 4 ++-
5 files changed, 43 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f3e3aa82/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
index 0745d5d..978d975 100644
--- a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
@@ -15,7 +15,7 @@
;; limitations under the License.
(ns backtype.storm.daemon.logviewer
(:use compojure.core)
- (:use [clojure.set :only [difference]])
+ (:use [clojure.set :only [difference intersection]])
(:use [clojure.string :only [blank?]])
(:use [hiccup core page-helpers])
(:use [backtype.storm config util log timer])
@@ -175,19 +175,32 @@
(recur)))))
(.toString output)))))
-(defn get-log-user-whitelist [fname]
+(defn get-log-user-group-whitelist [fname]
(let [wl-file (get-log-metadata-file fname)
- m (clojure-from-yaml-file wl-file)]
- (if-let [whitelist (.get m LOGS-USERS)] whitelist [])))
+ m (clojure-from-yaml-file wl-file)
+ user-wl (.get m LOGS-USERS)
+ user-wl (if user-wl user-wl [])
+ group-wl (.get m LOGS-GROUPS)
+ group-wl (if group-wl group-wl [])]
+ [user-wl group-wl]))
+
+(def igroup-mapper (AuthUtils/GetGroupMappingServiceProviderPlugin *STORM-CONF*))
+(defn user-groups
+ [user]
+ (if (blank? user) [] (.getGroups igroup-mapper user)))
(defn authorized-log-user? [user fname conf]
(if (or (blank? user) (blank? fname))
nil
- (let [whitelist (get-log-user-whitelist fname)
+ (let [groups (user-groups fname)
+ [user-wl group-wl] (get-log-user-group-whitelist fname)
logs-users (concat (conf LOGS-USERS)
(conf NIMBUS-ADMINS)
- whitelist)]
- (some #(= % user) logs-users))))
+ user-wl)
+ logs-groups (concat (conf LOGS-GROUPS)
+ group-wl)]
+ (or (some #(= % user) logs-users)
+ (< 0 (.size (intersection (set groups) (set group-wl))))))))
(defn log-root-dir
"Given an appender name, as configured, get the parent directory of the appender's log file.
http://git-wip-us.apache.org/repos/asf/storm/blob/f3e3aa82/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index d82fd12..8a5c039 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -563,6 +563,10 @@
(defn write-log-metadata! [storm-conf user worker-id storm-id port conf]
(let [data {TOPOLOGY-SUBMITTER-USER user
"worker-id" worker-id
+ LOGS-GROUPS (sort (distinct (remove nil?
+ (concat
+ (storm-conf LOGS-GROUPS)
+ (storm-conf TOPOLOGY-GROUPS)))))
LOGS-USERS (sort (distinct (remove nil?
(concat
(storm-conf LOGS-USERS)
http://git-wip-us.apache.org/repos/asf/storm/blob/f3e3aa82/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 48e1822..78e2a30 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -491,6 +491,12 @@ public class Config extends HashMap<String, Object> {
public static final Object LOGS_USERS_SCHEMA = ConfigValidation.StringsValidator;
/**
+ * A list of groups allowed to view logs via the Log Viewer
+ */
+ public static final String LOGS_GROUPS = "logs.groups";
+ public static final Object LOGS_GROUPS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
* Appender name used by log viewer to determine log directory.
*/
public static final String LOGVIEWER_APPENDER_NAME = "logviewer.appender.name";
@@ -521,12 +527,6 @@ public class Config extends HashMap<String, Object> {
public static final Object UI_HEADER_BUFFER_BYTES_SCHEMA = Number.class;
/**
- * A list of users allowed to view topologies via the UI
- */
- public static final String UI_USERS = "ui.users";
- public static final Object UI_USERS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
* List of DRPC servers so that the DRPCSpout knows who to talk to.
*/
public static final String DRPC_SERVERS = "drpc.servers";
http://git-wip-us.apache.org/repos/asf/storm/blob/f3e3aa82/storm-core/test/clj/backtype/storm/logviewer_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/logviewer_test.clj b/storm-core/test/clj/backtype/storm/logviewer_test.clj
index 37e63b9..93f22e6 100644
--- a/storm-core/test/clj/backtype/storm/logviewer_test.clj
+++ b/storm-core/test/clj/backtype/storm/logviewer_test.clj
@@ -165,23 +165,28 @@
(deftest test-authorized-log-user
(testing "allow cluster admin"
(let [conf {NIMBUS-ADMINS ["alice"]}]
- (stubbing [logviewer/get-log-user-whitelist []]
+ (stubbing [logviewer/get-log-user-group-whitelist [[] []]]
(is (logviewer/authorized-log-user? "alice" "non-blank-fname" conf)))))
(testing "ignore any cluster-set topology.users"
(let [conf {TOPOLOGY-USERS ["alice"]}]
- (stubbing [logviewer/get-log-user-whitelist []]
+ (stubbing [logviewer/get-log-user-group-whitelist [[] []]]
(is (not (logviewer/authorized-log-user? "alice" "non-blank-fname" conf))))))
(testing "allow cluster logs user"
(let [conf {LOGS-USERS ["alice"]}]
- (stubbing [logviewer/get-log-user-whitelist []]
+ (stubbing [logviewer/get-log-user-group-whitelist [[] []]]
(is (logviewer/authorized-log-user? "alice" "non-blank-fname" conf)))))
(testing "allow whitelisted topology user"
- (stubbing [logviewer/get-log-user-whitelist ["alice"]]
+ (stubbing [logviewer/get-log-user-group-whitelist [["alice"] []]]
+ (is (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))))
+
+ (testing "allow whitelisted topology group"
+ (stubbing [logviewer/get-log-user-group-whitelist [[] ["alice-group"]]
+ logviewer/user-groups ["alice-group"]]
(is (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))))
(testing "disallow user not in nimbus admin, topo user, logs user, or whitelist"
- (stubbing [logviewer/get-log-user-whitelist []]
+ (stubbing [logviewer/get-log-user-group-whitelist [[] []]]
(is (not (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))))))
http://git-wip-us.apache.org/repos/asf/storm/blob/f3e3aa82/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index a3594a3..0785475 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -494,12 +494,14 @@
exp-storm-id "0123456789"
exp-port 4242
exp-logs-users ["bob" "charlie" "daryl"]
+ exp-logs-groups []
storm-conf {TOPOLOGY-SUBMITTER-USER "alice"
TOPOLOGY-USERS ["charlie" "bob"]
LOGS-USERS ["daryl"]}
exp-data {TOPOLOGY-SUBMITTER-USER exp-owner
"worker-id" exp-worker-id
- LOGS-USERS exp-logs-users}
+ LOGS-USERS exp-logs-users
+ LOGS-GROUPS exp-logs-groups}
conf {}]
(mocking [supervisor/write-log-metadata-to-yaml-file!]
(supervisor/write-log-metadata! storm-conf exp-owner exp-worker-id
[3/7] storm git commit: Updated some of the tests.
Posted by bo...@apache.org.
Updated some of the tests.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/618e4851
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/618e4851
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/618e4851
Branch: refs/heads/master
Commit: 618e485110350ca1f6943e717632ab5eb20430e6
Parents: 68f4bcc
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Nov 20 09:49:55 2014 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Nov 20 09:49:55 2014 -0600
----------------------------------------------------------------------
storm-core/test/clj/backtype/storm/logviewer_test.clj | 7 ++++---
storm-core/test/clj/backtype/storm/supervisor_test.clj | 4 +++-
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/618e4851/storm-core/test/clj/backtype/storm/logviewer_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/logviewer_test.clj b/storm-core/test/clj/backtype/storm/logviewer_test.clj
index 0600903..23fa0b9 100644
--- a/storm-core/test/clj/backtype/storm/logviewer_test.clj
+++ b/storm-core/test/clj/backtype/storm/logviewer_test.clj
@@ -171,10 +171,11 @@
(verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
(verify-first-call-args-for logviewer/user-groups "alice"))))
- (testing "ignore any cluster-set topology.users"
- (let [conf {TOPOLOGY-USERS ["alice"]}]
+ (testing "ignore any cluster-set topology.users topology.groups"
+ (let [conf {TOPOLOGY-USERS ["alice"]
+ TOPOLOGY-GROUPS ["alice-group"]}]
(stubbing [logviewer/get-log-user-group-whitelist [[] []]
- logviewer/user-groups []]
+ logviewer/user-groups ["alice-group"]]
(is (not (logviewer/authorized-log-user? "alice" "non-blank-fname" conf)))
(verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
(verify-first-call-args-for logviewer/user-groups "alice"))))
http://git-wip-us.apache.org/repos/asf/storm/blob/618e4851/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index 0785475..b730cb3 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -494,9 +494,11 @@
exp-storm-id "0123456789"
exp-port 4242
exp-logs-users ["bob" "charlie" "daryl"]
- exp-logs-groups []
+ exp-logs-groups ["read-only-group" "special-group"]
storm-conf {TOPOLOGY-SUBMITTER-USER "alice"
TOPOLOGY-USERS ["charlie" "bob"]
+ TOPOLOGY-GROUPS ["special-group"]
+ LOGS-GROUPS ["read-only-group"]
LOGS-USERS ["daryl"]}
exp-data {TOPOLOGY-SUBMITTER-USER exp-owner
"worker-id" exp-worker-id
[5/7] storm git commit: Updated SECURITY.md to reflect the change to
logs.groups.
Posted by bo...@apache.org.
Updated SECURITY.md to reflect the change to logs.groups.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/231ac22b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/231ac22b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/231ac22b
Branch: refs/heads/master
Commit: 231ac22b7caa254d96e2647e784747067d52cd38
Parents: cdc699e
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Dec 22 14:59:38 2014 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Dec 22 14:59:38 2014 -0600
----------------------------------------------------------------------
SECURITY.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/231ac22b/SECURITY.md
----------------------------------------------------------------------
diff --git a/SECURITY.md b/SECURITY.md
index a5cd264..02dfa9d 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -274,9 +274,9 @@ The *SimpleACLAuthorizer* plug-in needs to know who the supervisor users are, an
These are set through *nimbus.supervisor.users* and *nimbus.admins* respectively. Each can either be a full Kerberos principal name, or the name of the user with host and realm stripped off.
-The UI and Log servers have their own authorization configurations. These are set through *logs.users* and *ui.users*. These should be set to the admin users for all of the nodes in the cluster.
+The Log servers have their own authorization configurations. These are set through *logs.users* and *logs.groups*. These should be set to the admin users or groups for all of the nodes in the cluster.
-When a topology is sumbitted, the sumbitting user can specify users in this list as well. The users specified-in addition to the users in the cluster-wide setting-will be granted access to the submitted topology's details in the ui and/or to the topology's worker logs in the logviewers.
+When a topology is sumbitted, the sumbitting user can specify users in this list as well. The users and groups specified-in addition to the users in the cluster-wide setting-will be granted access to the submitted topology's worker logs in the logviewers.
### Supervisors headless User and group Setup