You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2017/09/18 13:58:49 UTC
[1/2] storm git commit: [STORM-2741] Add in config options for
metrics consumer cpu and memory
Repository: storm
Updated Branches:
refs/heads/master 124acb92d -> 05a74c73d
[STORM-2741] Add in config options for metrics consumer cpu and memory
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4a29b57a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4a29b57a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4a29b57a
Branch: refs/heads/master
Commit: 4a29b57a1c58cc6fda54356ae5568843d1abbb3c
Parents: 124acb9
Author: Ethan Li <et...@gmail.com>
Authored: Fri Sep 15 09:45:34 2017 -0500
Committer: Ethan Li <et...@gmail.com>
Committed: Fri Sep 15 09:55:34 2017 -0500
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/Config.java | 22 ++++++++++++++++++++
.../src/jvm/org/apache/storm/Constants.java | 2 +-
.../org/apache/storm/daemon/StormCommon.java | 1 +
.../apache/storm/scheduler/TopologyDetails.java | 14 +++++++++++++
4 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/4a29b57a/storm-client/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/Config.java b/storm-client/src/jvm/org/apache/storm/Config.java
index 61c75e6..5623698 100644
--- a/storm-client/src/jvm/org/apache/storm/Config.java
+++ b/storm-client/src/jvm/org/apache/storm/Config.java
@@ -249,6 +249,28 @@ public class Config extends HashMap<String, Object> {
public static final String TOPOLOGY_ACKER_CPU_PCORE_PERCENT = "topology.acker.cpu.pcore.percent";
/**
+ * The maximum amount of memory an instance of a metrics consumer will take on heap. This enables the scheduler
+ * to allocate slots on machines with enough available memory. A default value will be set for this config if user does not override
+ */
+ @isPositiveNumber(includeZero = true)
+ public static final String TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB = "topology.metrics.consumer.resources.onheap.memory.mb";
+
+ /**
+ * The maximum amount of memory an instance of a metrics consumer will take off heap. This enables the scheduler
+ * to allocate slots on machines with enough available memory. A default value will be set for this config if user does not override
+ */
+ @isPositiveNumber(includeZero = true)
+ public static final String TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB = "topology.metrics.consumer.resources.offheap.memory.mb";
+
+ /**
+ * The config indicates the percentage of cpu for a core an instance(executor) of a metrics consumer will use.
+ * Assuming the a core value to be 100, a value of 10 indicates 10% of the core.
+ * The P in PCORE represents the term "physical". A default value will be set for this config if user does not override
+ */
+ @isPositiveNumber(includeZero = true)
+ public static final String TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT = "topology.metrics.consumer.cpu.pcore.percent";
+
+ /**
* The class name of the {@link org.apache.storm.state.StateProvider} implementation. If not specified
* defaults to {@link org.apache.storm.state.InMemoryKeyValueStateProvider}. This can be overridden
* at the component level.
http://git-wip-us.apache.org/repos/asf/storm/blob/4a29b57a/storm-client/src/jvm/org/apache/storm/Constants.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/Constants.java b/storm-client/src/jvm/org/apache/storm/Constants.java
index b86b1ca..7375c70 100644
--- a/storm-client/src/jvm/org/apache/storm/Constants.java
+++ b/storm-client/src/jvm/org/apache/storm/Constants.java
@@ -30,7 +30,7 @@ public class Constants {
public static final List<Long> SYSTEM_EXECUTOR_ID = Arrays.asList(-1L, -1L);
public static final String SYSTEM_COMPONENT_ID = "__system";
public static final String SYSTEM_TICK_STREAM_ID = "__tick";
- public static final String METRICS_COMPONENT_ID_PREFIX = "__metrics";
+ public static final String METRICS_COMPONENT_ID_PREFIX = "__metrics_";
public static final String METRICS_STREAM_ID = "__metrics";
public static final String METRICS_TICK_STREAM_ID = "__metrics_tick";
public static final String CREDENTIALS_CHANGED_STREAM_ID = "__credentials";
http://git-wip-us.apache.org/repos/asf/storm/blob/4a29b57a/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java b/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java
index 158c2eb..617e7ae 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java
@@ -419,6 +419,7 @@ public class StormCommon {
classOccurrencesMap.put(className, occurrenceNum);
id = Constants.METRICS_COMPONENT_ID_PREFIX + className + "#" + occurrenceNum;
} else {
+ id = Constants.METRICS_COMPONENT_ID_PREFIX + className;
classOccurrencesMap.put(className, 1);
}
metricsConsumerBolts.put(id, metricsConsumerBolt);
http://git-wip-us.apache.org/repos/asf/storm/blob/4a29b57a/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java b/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
index fb1b0f7..005df7d 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.storm.Config;
+import org.apache.storm.Constants;
import org.apache.storm.daemon.Acker;
import org.apache.storm.generated.Bolt;
import org.apache.storm.generated.ComponentCommon;
@@ -508,6 +509,19 @@ public class TopologyDetails {
resourceListForExec.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT,
ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT)));
}
+ } else if (component.startsWith(Constants.METRICS_COMPONENT_ID_PREFIX)) {
+ if (topologyConf.containsKey(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB)) {
+ resourceListForExec.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB)));
+ }
+ if (topologyConf.containsKey(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB)) {
+ resourceListForExec.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB)));
+ }
+ if (topologyConf.containsKey(Config.TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT)) {
+ resourceListForExec.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT,
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT)));
+ }
}
}
[2/2] storm git commit: Merge branch 'STORM-2741' of
https://github.com/Ethanlm/storm into STORM-2741
Posted by bo...@apache.org.
Merge branch 'STORM-2741' of https://github.com/Ethanlm/storm into STORM-2741
STORM-2741: Add in config options for metrics consumer cpu and memory
This closes #2326
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/05a74c73
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/05a74c73
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/05a74c73
Branch: refs/heads/master
Commit: 05a74c73db9b9017d3672d3cc055e8f1ada1d69e
Parents: 124acb9 4a29b57
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Mon Sep 18 08:41:13 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 18 08:41:13 2017 -0500
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/Config.java | 22 ++++++++++++++++++++
.../src/jvm/org/apache/storm/Constants.java | 2 +-
.../org/apache/storm/daemon/StormCommon.java | 1 +
.../apache/storm/scheduler/TopologyDetails.java | 14 +++++++++++++
4 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------