You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2015/04/01 05:43:15 UTC
[1/4] storm git commit: STORM-615. Add REST API to upload topology.
Repository: storm
Updated Branches:
refs/heads/master db6df0caf -> cd6206320
STORM-615. Add REST API to upload topology.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1945f709
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1945f709
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1945f709
Branch: refs/heads/master
Commit: 1945f709174bcd867c8412198c559bbb15e287d9
Parents: 117256b
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Thu Mar 12 13:58:31 2015 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Tue Mar 31 17:21:33 2015 -0700
----------------------------------------------------------------------
STORM-UI-REST-API.md | 31 ++++++++++++
storm-core/src/clj/backtype/storm/ui/core.clj | 58 +++++++++++++++++++++-
2 files changed, 87 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1945f709/STORM-UI-REST-API.md
----------------------------------------------------------------------
diff --git a/STORM-UI-REST-API.md b/STORM-UI-REST-API.md
index 72b07d4..cdbf05e 100644
--- a/STORM-UI-REST-API.md
+++ b/STORM-UI-REST-API.md
@@ -616,6 +616,37 @@ error response:
}
```
+### /api/v1/uploadTopology (POST)
+
+uploads a topology.
+
+Caution: This api doesn't work in security mode.
+
+|Parameter |Value |Description |
+|----------|--------|-------------|
+|topologyConfig |String (required)| topology json config |
+|topologyJar |String (required)| topology jar file |
+
+Sample topologyConfig json:
+```json
+{"topologyMainClass": "storm.starter.WordCountTopology", "topologyMainClassArgs": ["wordcount1"]}
+```
+
+Examples:
+
+```no-highlight
+curl -i -b ~/cookiejar.txt -c ~/cookiejar.txt -X POST
+-H 'x-csrf-token: ycit8Wi89ZdAOo9KKaka/Pvd0vnx8TZzP8xSDDSw8J8bTfyn4jz38VN4Xcb7CF6xigRzDLaGVHbrSj80'
+-F topologyConfig='{"topologyMainClass": "storm.starter.WordCountTopology", "topologyMainClassArgs": ["wordcount1"]}'
+-F topologyJar=@examples/storm-starter/storm-starter-topologies-0.10.0-SNAPSHOT.jar
+http://localhost:8080/api/v1/uploadTopology
+```
+
+Sample Response:
+
+```json
+{"status":"success"}
+```
### /api/v1/topology/:id/activate (POST)
http://git-wip-us.apache.org/repos/asf/storm/blob/1945f709/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index c64f35d..0a797a3 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -16,7 +16,9 @@
(ns backtype.storm.ui.core
(:use compojure.core)
- (:use ring.middleware.reload)
+ (:use [clojure.java.shell :only [sh]])
+ (:use ring.middleware.reload
+ ring.middleware.multipart-params)
(:use [ring.middleware.json :only [wrap-json-params]])
(:use [hiccup core page-helpers])
(:use [backtype.storm config util log])
@@ -503,6 +505,39 @@
(hashmap-to-persistent bolts))
spout-comp-summs bolt-comp-summs window id))))
+(defn validate-tplg-submit-params [params]
+ (let [tplg-jar-file (params :topologyJar)
+ tplg-config (if (not-nil? (params :topologyConfig)) (from-json (params :topologyConfig)))]
+ (cond
+ (nil? tplg-jar-file) {:valid false :error "missing topology jar file"}
+ (nil? tplg-config) {:valid false :error "missing topology config"}
+ (nil? (tplg-config "topologyMainClass")) {:valid false :error "topologyMainClass missing in topologyConfig"}
+ :else {:valid true})))
+
+(defn run-tplg-submit-cmd [tplg-jar-file tplg-config user]
+ (let [tplg-main-class (if (not-nil? tplg-config) (trim (tplg-config "topologyMainClass")))
+ tplg-main-class-args (if (not-nil? tplg-config) (clojure.string/join " " (tplg-config "topologyMainClassArgs")))
+ tplg-jvm-opts (if (not-nil? tplg-config) (clojure.string/join " " (tplg-config "topologyJvmOpts")))
+ storm-home (System/getProperty "storm.home")
+ storm-conf-dir (str storm-home file-path-separator "conf")
+ storm-log-dir (if (not-nil? (*STORM-CONF* "storm.log.dir")) (*STORM-CONF* "storm.log.dir")
+ (str storm-home file-path-separator "logs"))
+ storm-libs (str storm-home file-path-separator "lib" file-path-separator "*")
+ java-cmd (str (System/getProperty "java.home") file-path-separator "bin" file-path-separator "java")
+ storm-cmd (str storm-home file-path-separator "bin" file-path-separator "storm")
+ tplg-cmd-response (sh storm-cmd "jar" tplg-jar-file
+ tplg-main-class
+ tplg-main-class-args
+ (if (not= user "unknown") (str "-c storm.doAsUser=" user) ""))]
+ (log-message "tplg-cmd-response " tplg-cmd-response)
+ (cond
+ (= (tplg-cmd-response :exit) 0) {"status" "success"}
+ (and (not= (tplg-cmd-response :exit) 0)
+ (not-nil? (re-find #"already exists on cluster" (tplg-cmd-response :err)))) {"status" "failed" "error" "Topology with the same name exists in cluster"}
+ (not= (tplg-cmd-response :exit) 0) {"status" "failed" "error" (clojure.string/trim-newline (tplg-cmd-response :err))}
+ :else {"status" "success" "response" "topology deployed"}
+ )))
+
(defn cluster-configuration []
(with-nimbus nimbus
(.getNimbusConf ^Nimbus$Client nimbus)))
@@ -982,7 +1017,25 @@
(.killTopologyWithOpts nimbus name options)
(log-message "Killing topology '" name "' with wait time: " wait-time " secs")))
(json-response (topology-op-response id "kill") (m "callback")))
-
+ (POST "/api/v1/uploadTopology" [:as {:keys [cookies servlet-request]} id & params]
+ (assert-authorized-user servlet-request "submitTopology")
+ (let [valid-tplg (validate-tplg-submit-params params)
+ valid (valid-tplg :valid)
+ context (ReqContext/context)]
+ (if http-creds-handler (.populateContext http-creds-handler context servlet-request))
+ (if (= valid true)
+ (let [tplg-file-data (params :topologyJar)
+ tplg-temp-file (tplg-file-data :tempfile)
+ tplg-file-name (tplg-file-data :filename)
+ tplg-jar-file (clojure.string/join [(.getParent tplg-temp-file) file-path-separator tplg-file-name])
+ tplg-config (if (not-nil? (params :topologyConfig)) (from-json (params :topologyConfig)))
+ principal (if (.isImpersonating context) (.realPrincipal context) (.principal context))
+ user (if principal (.getName principal) "unknown")]
+ (.renameTo tplg-temp-file (File. tplg-jar-file))
+ (let [ret (run-tplg-submit-cmd tplg-jar-file tplg-config user)]
+ (json-response ret (params "callback"))))
+ (json-response {"status" "failed" "error" (valid-tplg :error)} (params "callback"))
+ )))
(GET "/" [:as {cookies :cookies}]
(resp/redirect "/index.html"))
(route/resources "/")
@@ -1012,6 +1065,7 @@
(def app
(handler/site (-> main-routes
(wrap-json-params)
+ (wrap-multipart-params)
(wrap-reload '[backtype.storm.ui.core])
(wrap-anti-forgery {:error-response csrf-error-response})
catch-errors)))
[4/4] storm git commit: Added STORM-615 to CHANGELOG.
Posted by sr...@apache.org.
Added STORM-615 to CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cd620632
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cd620632
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cd620632
Branch: refs/heads/master
Commit: cd62063206e4f888f35510a792cdbdf18582e533
Parents: 7b1f3f6
Author: Sriharsha Chintalapani <ha...@hortonworks.com>
Authored: Tue Mar 31 20:37:14 2015 -0700
Committer: Sriharsha Chintalapani <ha...@hortonworks.com>
Committed: Tue Mar 31 20:37:14 2015 -0700
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/cd620632/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a81810e..d11d5a6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM:615: Add REST API to upload topology.
* STORM-741: Allow users to pass a config value to perform impersonation.
* STORM-724: Document RedisStoreBolt and RedisLookupBolt which is missed.
* STORM-711: All connectors should use collector.reportError and tuple anchoring.
[2/4] storm git commit: STORM-615. Add REST API to upload topology.
Posted by sr...@apache.org.
STORM-615. Add REST API to upload topology.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/002bf003
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/002bf003
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/002bf003
Branch: refs/heads/master
Commit: 002bf003f6b014c1a4f8b83de0a21165691f138b
Parents: 1945f70
Author: Sriharsha Chintalapani <ma...@harsha.io>
Authored: Tue Mar 31 17:24:47 2015 -0700
Committer: Sriharsha Chintalapani <ma...@harsha.io>
Committed: Tue Mar 31 17:24:47 2015 -0700
----------------------------------------------------------------------
STORM-UI-REST-API.md | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/002bf003/STORM-UI-REST-API.md
----------------------------------------------------------------------
diff --git a/STORM-UI-REST-API.md b/STORM-UI-REST-API.md
index cdbf05e..808344b 100644
--- a/STORM-UI-REST-API.md
+++ b/STORM-UI-REST-API.md
@@ -620,7 +620,6 @@ error response:
uploads a topology.
-Caution: This api doesn't work in security mode.
|Parameter |Value |Description |
|----------|--------|-------------|
[3/4] storm git commit: Merge branch 'STORM-615-V2' of
https://github.com/harshach/incubator-storm into STORM-615
Posted by sr...@apache.org.
Merge branch 'STORM-615-V2' of https://github.com/harshach/incubator-storm into STORM-615
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7b1f3f67
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7b1f3f67
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7b1f3f67
Branch: refs/heads/master
Commit: 7b1f3f67619d86394e628f984ebb784adec86f2d
Parents: db6df0c 002bf00
Author: Sriharsha Chintalapani <ha...@hortonworks.com>
Authored: Tue Mar 31 20:36:43 2015 -0700
Committer: Sriharsha Chintalapani <ha...@hortonworks.com>
Committed: Tue Mar 31 20:36:43 2015 -0700
----------------------------------------------------------------------
STORM-UI-REST-API.md | 30 +++++++++++
storm-core/src/clj/backtype/storm/ui/core.clj | 58 +++++++++++++++++++++-
2 files changed, 86 insertions(+), 2 deletions(-)
----------------------------------------------------------------------