You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by kn...@apache.org on 2015/08/13 18:19:01 UTC
[1/5] storm git commit: STORM-964,
add config for worker logwriter to restrict its mem usage
Repository: storm
Updated Branches:
refs/heads/master 9e809035c -> 0ce410684
STORM-964, add config for worker logwriter to restrict its mem usage
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f5ce1f86
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f5ce1f86
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f5ce1f86
Branch: refs/heads/master
Commit: f5ce1f865aaf8fbebd364799dcc9685a96dd7f16
Parents: c3a9385
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Jul 29 10:54:56 2015 -0500
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Wed Jul 29 10:54:56 2015 -0500
----------------------------------------------------------------------
conf/defaults.yaml | 1 +
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 2 ++
storm-core/src/jvm/backtype/storm/Config.java | 6 ++++++
3 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f5ce1f86/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index c3fa372..ddc1c26 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -181,6 +181,7 @@ topology.stats.sample.rate: 0.05
topology.builtin.metrics.bucket.size.secs: 60
topology.fall.back.on.java.serialization: true
topology.worker.childopts: null
+topology.worker.lw.childopts: "-Xmx64m"
topology.executor.receive.buffer.size: 1024 #batched
topology.executor.send.buffer.size: 1024 #individual messages
topology.transfer.buffer.size: 1024 # batched
http://git-wip-us.apache.org/repos/asf/storm/blob/f5ce1f86/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index be8f682..cae56f6 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -675,6 +675,7 @@
(add-to-classpath topo-classpath))
top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS)
gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port)
+ topo-worker-lw-childopts (conf TOPOLOGY-WORKER-LW-CHILDOPTS)
user (storm-conf TOPOLOGY-SUBMITTER-USER)
logging-sensitivity (storm-conf TOPOLOGY-LOGGING-SENSITIVITY "S3")
logfilename (logs-filename storm-id port)
@@ -687,6 +688,7 @@
{"LD_LIBRARY_PATH" jlp})
command (concat
[(java-cmd) "-cp" classpath
+ topo-worker-lw-childopts
(str "-Dlogfile.name=" logfilename)
(str "-Dstorm.home=" storm-home)
(str "-Dstorm.id=" storm-id)
http://git-wip-us.apache.org/repos/asf/storm/blob/f5ce1f86/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 58e2a4a..7472e70 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -1221,6 +1221,12 @@ public class Config extends HashMap<String, Object> {
public static final Object TOPOLOGY_WORKER_GC_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
/**
+ * Topology-specific options for the logwriter process of a worker.
+ */
+ public static final String TOPOLOGY_WORKER_LW_CHILDOPTS="topology.worker.lw.childopts";
+ public static final Object TOPOLOGY_WORKER_LW_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
+
+ /**
* Topology-specific classpath for the worker child process. This is combined to the usual classpath.
*/
public static final String TOPOLOGY_CLASSPATH="topology.classpath";
[5/5] storm git commit: Updating Changelog for STORM-964
Posted by kn...@apache.org.
Updating Changelog for STORM-964
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0ce41068
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0ce41068
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0ce41068
Branch: refs/heads/master
Commit: 0ce410684c296209a4913ada36a428e009bc4ce4
Parents: b84981e
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Thu Aug 13 11:18:41 2015 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Thu Aug 13 11:18:41 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0ce41068/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d44e684..2045668 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-964: Add config (with small default value) for logwriter to restrict its memory usage
* STORM-980: Re-include storm-kafka tests from Travis CI build
* STORM-960: HiveBolt should ack tuples only after flushing.
* STORM-951: Storm Hive connector leaking connections.
[2/5] storm git commit: Change lw to logwriter
Posted by kn...@apache.org.
Change lw to logwriter
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d9d2bb73
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d9d2bb73
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d9d2bb73
Branch: refs/heads/master
Commit: d9d2bb7321165c26c256a1d3defdd8507191a6b4
Parents: f5ce1f8
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Jul 29 17:30:24 2015 -0500
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Wed Jul 29 17:30:24 2015 -0500
----------------------------------------------------------------------
conf/defaults.yaml | 2 +-
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 4 ++--
storm-core/src/jvm/backtype/storm/Config.java | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d9d2bb73/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index ddc1c26..c44709f 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -181,7 +181,7 @@ topology.stats.sample.rate: 0.05
topology.builtin.metrics.bucket.size.secs: 60
topology.fall.back.on.java.serialization: true
topology.worker.childopts: null
-topology.worker.lw.childopts: "-Xmx64m"
+topology.worker.logwriter.childopts: "-Xmx64m"
topology.executor.receive.buffer.size: 1024 #batched
topology.executor.send.buffer.size: 1024 #individual messages
topology.transfer.buffer.size: 1024 # batched
http://git-wip-us.apache.org/repos/asf/storm/blob/d9d2bb73/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index cae56f6..eb2a06a 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -675,7 +675,7 @@
(add-to-classpath topo-classpath))
top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS)
gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port)
- topo-worker-lw-childopts (conf TOPOLOGY-WORKER-LW-CHILDOPTS)
+ topo-worker-logwriter-childopts (conf TOPOLOGY-WORKER-LOGWRITER-CHILDOPTS)
user (storm-conf TOPOLOGY-SUBMITTER-USER)
logging-sensitivity (storm-conf TOPOLOGY-LOGGING-SENSITIVITY "S3")
logfilename (logs-filename storm-id port)
@@ -688,7 +688,7 @@
{"LD_LIBRARY_PATH" jlp})
command (concat
[(java-cmd) "-cp" classpath
- topo-worker-lw-childopts
+ topo-worker-logwriter-childopts
(str "-Dlogfile.name=" logfilename)
(str "-Dstorm.home=" storm-home)
(str "-Dstorm.id=" storm-id)
http://git-wip-us.apache.org/repos/asf/storm/blob/d9d2bb73/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 7472e70..699641c 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -1223,8 +1223,8 @@ public class Config extends HashMap<String, Object> {
/**
* Topology-specific options for the logwriter process of a worker.
*/
- public static final String TOPOLOGY_WORKER_LW_CHILDOPTS="topology.worker.lw.childopts";
- public static final Object TOPOLOGY_WORKER_LW_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
+ public static final String TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS="topology.worker.logwriter.childopts";
+ public static final Object TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
/**
* Topology-specific classpath for the worker child process. This is combined to the usual classpath.
[3/5] storm git commit: use storm-conf
Posted by kn...@apache.org.
use storm-conf
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/eb6d7383
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/eb6d7383
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/eb6d7383
Branch: refs/heads/master
Commit: eb6d7383861220eaeb685206d3946899c20506ac
Parents: d9d2bb7
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Jul 29 22:18:39 2015 -0500
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Wed Jul 29 22:18:39 2015 -0500
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/eb6d7383/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index eb2a06a..ce638ce 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -675,7 +675,7 @@
(add-to-classpath topo-classpath))
top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS)
gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port)
- topo-worker-logwriter-childopts (conf TOPOLOGY-WORKER-LOGWRITER-CHILDOPTS)
+ topo-worker-logwriter-childopts (storm-conf TOPOLOGY-WORKER-LOGWRITER-CHILDOPTS)
user (storm-conf TOPOLOGY-SUBMITTER-USER)
logging-sensitivity (storm-conf TOPOLOGY-LOGGING-SENSITIVITY "S3")
logfilename (logs-filename storm-id port)
[4/5] storm git commit: Merge branch '964' of
https://github.com/zhuoliu/storm
Posted by kn...@apache.org.
Merge branch '964' of https://github.com/zhuoliu/storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b84981ec
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b84981ec
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b84981ec
Branch: refs/heads/master
Commit: b84981ec98af8aac0f361fe886f903380171e619
Parents: 9e80903 eb6d738
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Thu Aug 13 11:16:41 2015 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Thu Aug 13 11:16:41 2015 -0500
----------------------------------------------------------------------
conf/defaults.yaml | 1 +
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 2 ++
storm-core/src/jvm/backtype/storm/Config.java | 6 ++++++
3 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b84981ec/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------