You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/06/04 20:59:39 UTC
[1/3] storm git commit: STORM-760: Use JSON for serialized conf
Repository: storm
Updated Branches:
refs/heads/master 19e3a28e1 -> bf2ef232f
STORM-760: Use JSON for serialized conf
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5f55c51a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5f55c51a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5f55c51a
Branch: refs/heads/master
Commit: 5f55c51acd06697e339a6d2d5a07e46a21f3e0ab
Parents: 2154048
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Apr 7 14:15:09 2015 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Jun 4 08:58:41 2015 -0500
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/config.clj | 2 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 10 ++++----
.../src/jvm/backtype/storm/utils/Utils.java | 26 ++++++++++++++++++++
3 files changed, 32 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5f55c51a/storm-core/src/clj/backtype/storm/config.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/config.clj b/storm-core/src/clj/backtype/storm/config.clj
index a345daf..7647906 100644
--- a/storm-core/src/clj/backtype/storm/config.clj
+++ b/storm-core/src/clj/backtype/storm/config.clj
@@ -212,7 +212,7 @@
(let [stormroot (supervisor-stormdist-root conf storm-id)
conf-path (supervisor-stormconf-path stormroot)
topology-path (supervisor-stormcode-path stormroot)]
- (merge conf (clojurify-structure (Utils/javaDeserialize (FileUtils/readFileToByteArray (File. conf-path)) java.util.Map)))))
+ (merge conf (clojurify-structure (Utils/fromCompressedJsonConf (FileUtils/readFileToByteArray (File. conf-path)))))))
(defn read-supervisor-topology
[conf storm-id]
http://git-wip-us.apache.org/repos/asf/storm/blob/5f55c51a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index e079ba4..398d429 100644
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@ -108,10 +108,10 @@
(defn- read-storm-conf [conf storm-id]
(let [stormroot (master-stormdist-root conf storm-id)]
(merge conf
- (Utils/javaDeserialize
- (FileUtils/readFileToByteArray
- (File. (master-stormconf-path stormroot))
- ) java.util.Map))))
+ (clojurify-structure
+ (Utils/fromCompressedJsonConf
+ (FileUtils/readFileToByteArray
+ (File. (master-stormconf-path stormroot))))))))
(declare delay-event)
(declare mk-assignments)
@@ -323,7 +323,7 @@
(FileUtils/cleanDirectory (File. stormroot))
(setup-jar conf tmp-jar-location stormroot)
(FileUtils/writeByteArrayToFile (File. (master-stormcode-path stormroot)) (Utils/serialize topology))
- (FileUtils/writeByteArrayToFile (File. (master-stormconf-path stormroot)) (Utils/javaSerialize storm-conf))
+ (FileUtils/writeByteArrayToFile (File. (master-stormconf-path stormroot)) (Utils/toCompressedJsonConf storm-conf))
))
(defn- read-storm-topology [conf storm-id]
http://git-wip-us.apache.org/repos/asf/storm/blob/5f55c51a/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 66e199a..d83b926 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -136,6 +136,32 @@ public class Utils {
}
}
+ public static byte[] toCompressedJsonConf(Map<String, Object> stormConf) {
+ try {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ OutputStreamWriter out = new OutputStreamWriter(new GZIPOutputStream(bos));
+ JSONValue.writeJSONString(stormConf, out);
+ out.close();
+ return bos.toByteArray();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static Map<String, Object> fromCompressedJsonConf(byte[] serialized) {
+ try {
+ ByteArrayInputStream bis = new ByteArrayInputStream(serialized);
+ InputStreamReader in = new InputStreamReader(new GZIPInputStream(bis));
+ Object ret = JSONValue.parseWithException(in);
+ in.close();
+ return (Map<String,Object>)ret;
+ } catch(IOException ioe) {
+ throw new RuntimeException(ioe);
+ } catch(ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
public static <T> String join(Iterable<T> coll, String sep) {
Iterator<T> it = coll.iterator();
String ret = "";
[3/3] storm git commit: Aded STORM-760 to Changelog
Posted by bo...@apache.org.
Aded STORM-760 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bf2ef232
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bf2ef232
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bf2ef232
Branch: refs/heads/master
Commit: bf2ef232f4b2cd58a3a91801ac6d291a60d2cd38
Parents: 07f5e67
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Jun 4 13:51:55 2015 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Jun 4 13:51:55 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bf2ef232/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 83ae5a8..549eee3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,7 @@
## 0.11.0
## 0.10.0
+ * STORM-760: Use JSON for serialized conf
* STORM-833: Logging framework logback -> log4j 2.x
* STORM-842: Drop Support for Java 1.6
* STORM-835: Netty Client hold batch object until io operation complete
[2/3] storm git commit: Merge branch 'STORM-760' of
https://github.com/revans2/incubator-storm into STORM-760
Posted by bo...@apache.org.
Merge branch 'STORM-760' of https://github.com/revans2/incubator-storm into STORM-760
STORM-760: Use JSON for serialized conf
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/07f5e676
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/07f5e676
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/07f5e676
Branch: refs/heads/master
Commit: 07f5e6769ea35f8d680f937b3358176000ae8bdc
Parents: 19e3a28 5f55c51
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Jun 4 13:51:40 2015 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Jun 4 13:51:40 2015 -0500
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/config.clj | 2 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 10 ++++----
.../src/jvm/backtype/storm/utils/Utils.java | 26 ++++++++++++++++++++
3 files changed, 32 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/07f5e676/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------