You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2017/09/13 16:47:07 UTC
[1/2] storm git commit: [STORM-2730] Add in config options for acker
cpu and memory
Repository: storm
Updated Branches:
refs/heads/master c365bcae0 -> 79010512d
[STORM-2730] Add in config options for acker cpu and memory
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/32be4e36
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/32be4e36
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/32be4e36
Branch: refs/heads/master
Commit: 32be4e3674b3fdd4db427a53f24d9cac3b98412e
Parents: 32389d7
Author: Ethan Li <et...@gmail.com>
Authored: Mon Sep 11 09:18:04 2017 -0500
Committer: Ethan Li <et...@gmail.com>
Committed: Tue Sep 12 14:28:51 2017 -0500
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/Config.java | 22 ++++++++++++
.../apache/storm/scheduler/TopologyDetails.java | 37 +++++++++++++++-----
2 files changed, 50 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/32be4e36/storm-client/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/Config.java b/storm-client/src/jvm/org/apache/storm/Config.java
index 13b8a22..61c75e6 100644
--- a/storm-client/src/jvm/org/apache/storm/Config.java
+++ b/storm-client/src/jvm/org/apache/storm/Config.java
@@ -227,6 +227,28 @@ public class Config extends HashMap<String, Object> {
public static final String TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT = "topology.component.cpu.pcore.percent";
/**
+ * The maximum amount of memory an instance of an acker will take on heap. This enables the scheduler
+ * to allocate slots on machines with enough available memory. A default value will be set for this config if user does not override
+ */
+ @isPositiveNumber(includeZero = true)
+ public static final String TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB = "topology.acker.resources.onheap.memory.mb";
+
+ /**
+ * The maximum amount of memory an instance of an acker will take off heap. This enables the scheduler
+ * to allocate slots on machines with enough available memory. A default value will be set for this config if user does not override
+ */
+ @isPositiveNumber(includeZero = true)
+ public static final String TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB = "topology.acker.resources.offheap.memory.mb";
+
+ /**
+ * The config indicates the percentage of cpu for a core an instance(executor) of an acker will use.
+ * Assuming the a core value to be 100, a value of 10 indicates 10% of the core.
+ * The P in PCORE represents the term "physical". A default value will be set for this config if user does not override
+ */
+ @isPositiveNumber(includeZero = true)
+ public static final String TOPOLOGY_ACKER_CPU_PCORE_PERCENT = "topology.acker.cpu.pcore.percent";
+
+ /**
* The class name of the {@link org.apache.storm.state.StateProvider} implementation. If not specified
* defaults to {@link org.apache.storm.state.InMemoryKeyValueStateProvider}. This can be overridden
* at the component level.
http://git-wip-us.apache.org/repos/asf/storm/blob/32be4e36/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java b/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
index 575463b..fb1b0f7 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.storm.Config;
+import org.apache.storm.daemon.Acker;
import org.apache.storm.generated.Bolt;
import org.apache.storm.generated.ComponentCommon;
import org.apache.storm.generated.ComponentType;
@@ -166,14 +167,6 @@ public class TopologyDetails {
// topology.getbolt (AKA sys tasks most specifically __acker tasks)
for (ExecutorDetails exec : getExecutors()) {
if (!resourceList.containsKey(exec)) {
- LOG.debug(
- "Scheduling {} {} with memory requirement as 'on heap' - {} and 'off heap' - {} and "
- + "CPU requirement as {}",
- getExecutorToComponent().get(exec),
- exec,
- topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB),
- topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB),
- topologyConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT));
addDefaultResforExec(exec);
}
}
@@ -482,17 +475,43 @@ public class TopologyDetails {
defaultResourceList.put(
Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
topologyComponentResourcesOnheapMemoryMb);
+
+ adjustResourcesForExec(exec, defaultResourceList);
+
LOG.debug(
- "Scheduling Executor: {} with memory requirement as onHeap: {} - offHeap: {} "
+ "Scheduling Executor: {} {} with memory requirement as onHeap: {} - offHeap: {} "
+ "and CPU requirement: {}",
+ getExecutorToComponent().get(exec),
exec,
topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB),
topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB),
topologyConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT));
+
addResourcesForExec(exec, defaultResourceList);
}
/**
+ * Some components might have different resource configs.
+ */
+ private void adjustResourcesForExec(ExecutorDetails exec, Map<String, Double> resourceListForExec) {
+ String component = getExecutorToComponent().get(exec);
+ if (component.equals(Acker.ACKER_COMPONENT_ID)) {
+ if (topologyConf.containsKey(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB)) {
+ resourceListForExec.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB)));
+ }
+ if (topologyConf.containsKey(Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB)) {
+ resourceListForExec.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB)));
+ }
+ if (topologyConf.containsKey(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT)) {
+ resourceListForExec.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT,
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT)));
+ }
+ }
+ }
+
+ /**
* initializes member variables
*/
private void initConfigs() {
[2/2] storm git commit: Merge branch 'STORM-2730' of
https://github.com/Ethanlm/storm into STORM-2730
Posted by bo...@apache.org.
Merge branch 'STORM-2730' of https://github.com/Ethanlm/storm into STORM-2730
STORM-2730: Add in config options for acker cpu and memory
This closes #2324
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/79010512
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/79010512
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/79010512
Branch: refs/heads/master
Commit: 79010512da01e156bb74478fce7062ac422e6475
Parents: c365bca 32be4e3
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Wed Sep 13 10:30:31 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Wed Sep 13 10:30:31 2017 -0500
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/Config.java | 22 ++++++++++++
.../apache/storm/scheduler/TopologyDetails.java | 37 +++++++++++++++-----
2 files changed, 50 insertions(+), 9 deletions(-)
----------------------------------------------------------------------