You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by je...@apache.org on 2016/02/18 06:02:59 UTC
[6/9] storm git commit: adding config
storm.cgroup.memory.limit.tolerance.margin.mb
adding config storm.cgroup.memory.limit.tolerance.margin.mb
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/42ce11ee
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/42ce11ee
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/42ce11ee
Branch: refs/heads/master
Commit: 42ce11ee1135c13f3efd075bb2d42f9ce995bc23
Parents: 0f77402
Author: Boyang Jerry Peng <je...@yahoo-inc.com>
Authored: Fri Feb 12 12:41:37 2016 -0600
Committer: Boyang Jerry Peng <je...@yahoo-inc.com>
Committed: Fri Feb 12 12:41:37 2016 -0600
----------------------------------------------------------------------
conf/defaults.yaml | 3 ++-
storm-core/src/clj/org/apache/storm/daemon/supervisor.clj | 6 ++++--
storm-core/src/jvm/org/apache/storm/Config.java | 5 +++++
3 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/42ce11ee/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index b88d478..166b249 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -263,7 +263,7 @@ topology.state.checkpoint.interval.ms: 1000
# topology priority describing the importance of the topology in decreasing importance starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as the priority number increases).
# Recommended range of 0-29 but no hard limit set.
topology.priority: 29
-topology.component.resources.onheap.memory.mb: 256.0
+topology.component.resources.onheap.memory.mb: 128.0
topology.component.resources.offheap.memory.mb: 0.0
topology.component.cpu.pcore.percent: 10.0
topology.worker.max.heap.size.mb: 768.0
@@ -298,3 +298,4 @@ storm.cgroup.hierarchy.name: "storm"
# Also determines whether the unit tests for cgroup runs. If cgroup.enable is set to false the unit tests for cgroups will not run
storm.supervisor.cgroup.rootdir: "storm"
storm.cgroup.cgexec.cmd: "/bin/cgexec"
+storm.cgroup.memory.limit.tolerance.margin.mb: 128.0
http://git-wip-us.apache.org/repos/asf/storm/blob/42ce11ee/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
index cb6bafc..dd29afe 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
@@ -1087,7 +1087,9 @@
(Utils/addToClasspath topo-classpath))
top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS)
- mem-onheap (int (Math/ceil (.get_mem_on_heap resources)))
+ mem-onheap (if (and (.get_mem_on_heap resources) (> (.get_mem_on_heap resources) 0)) ;; not nil and not zero
+ (int (Math/ceil (.get_mem_on_heap resources))) ;; round up
+ (storm-conf WORKER-HEAP-MEMORY-MB)) ;; otherwise use default value
mem-offheap (int (Math/ceil (.get_mem_off_heap resources)))
@@ -1160,7 +1162,7 @@
command (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE)
(do
(.reserveResourcesForWorker (:resource-isolation-manager supervisor) worker-id
- {"cpu" cpu "memory" (+ mem-onheap mem-offheap)})
+ {"cpu" cpu "memory" (+ mem-onheap mem-offheap (int (Math/ceil (conf STORM-CGROUP-MEMORY-MB-LIMIT-TOLERANCE-MARGIN))))})
(.getLaunchCommand (:resource-isolation-manager supervisor) worker-id
(java.util.ArrayList. (java.util.Arrays/asList (to-array command)))))
command)]
http://git-wip-us.apache.org/repos/asf/storm/blob/42ce11ee/storm-core/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/Config.java b/storm-core/src/jvm/org/apache/storm/Config.java
index ebe435c..a4d3409 100644
--- a/storm-core/src/jvm/org/apache/storm/Config.java
+++ b/storm-core/src/jvm/org/apache/storm/Config.java
@@ -2256,6 +2256,11 @@ public class Config extends HashMap<String, Object> {
@isString
public static String STORM_CGROUP_CGEXEC_CMD = "storm.cgroup.cgexec.cmd";
+ /**
+ * The amount of memory a worker can exceed its allocation before cgroup will kill it
+ */
+ @isPositiveNumber
+ public static String STORM_CGROUP_MEMORY_MB_LIMIT_TOLERANCE_MARGIN = "storm.cgroup.memory.limit.tolerance.margin.mb";
public static void setClasspath(Map conf, String cp) {
conf.put(Config.TOPOLOGY_CLASSPATH, cp);