You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/04/05 02:06:13 UTC
[1/3] storm git commit: [STORM-2363] Provide configuration to set the
number of RollingWindow
Repository: storm
Updated Branches:
refs/heads/master b2b9ecf27 -> c4170ab16
[STORM-2363] Provide configuration to set the number of RollingWindow
* rebased by @HeartSaVioR
* Closes #2037
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/de47d1bb
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/de47d1bb
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/de47d1bb
Branch: refs/heads/master
Commit: de47d1bb79ce1a14786725c38b85dc9694299ee9
Parents: b2b9ecf
Author: liuzhaokun <li...@zte.com.cn>
Authored: Sat Apr 1 08:56:21 2017 +0800
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Wed Apr 5 11:04:00 2017 +0900
----------------------------------------------------------------------
conf/defaults.yaml | 2 ++
storm-core/src/jvm/org/apache/storm/Config.java | 7 +++++++
.../src/jvm/org/apache/storm/executor/Executor.java | 8 ++++----
.../org/apache/storm/stats/BoltExecutorStats.java | 16 ++++++++--------
.../src/jvm/org/apache/storm/stats/CommonStats.java | 7 +++----
.../org/apache/storm/stats/SpoutExecutorStats.java | 10 +++++-----
6 files changed, 29 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/de47d1bb/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index cc9b82f..3ad7e9a 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -313,3 +313,5 @@ worker.metrics:
"CGroupMemoryLimit": "org.apache.storm.metric.cgroup.CGroupMemoryLimit"
"CGroupCpu": "org.apache.storm.metric.cgroup.CGroupCpu"
"CGroupCpuGuarantee": "org.apache.storm.metric.cgroup.CGroupCpuGuarantee"
+
+num.stat.buckets: 20
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/de47d1bb/storm-core/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/Config.java b/storm-core/src/jvm/org/apache/storm/Config.java
index 0ed070e..0a55ba7 100644
--- a/storm-core/src/jvm/org/apache/storm/Config.java
+++ b/storm-core/src/jvm/org/apache/storm/Config.java
@@ -2351,6 +2351,13 @@ public class Config extends HashMap<String, Object> {
@isPositiveNumber
public static String STORM_CGROUP_MEMORY_LIMIT_TOLERANCE_MARGIN_MB = "storm.cgroup.memory.limit.tolerance.margin.mb";
+ /**
+ * The number of Buckets
+ */
+ @isInteger
+ @isPositiveNumber
+ public static final String NUM_STAT_BUCKETS = "num.stat.buckets";
+
public static void setClasspath(Map conf, String cp) {
conf.put(Config.TOPOLOGY_CLASSPATH, cp);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/de47d1bb/storm-core/src/jvm/org/apache/storm/executor/Executor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/executor/Executor.java b/storm-core/src/jvm/org/apache/storm/executor/Executor.java
index cd39dc0..afe4bda 100644
--- a/storm-core/src/jvm/org/apache/storm/executor/Executor.java
+++ b/storm-core/src/jvm/org/apache/storm/executor/Executor.java
@@ -146,10 +146,10 @@ public abstract class Executor implements Callable, EventHandler<Object> {
Map<String, Bolt> bolts = topology.get_bolts();
if (spouts.containsKey(componentId)) {
this.type = StatsUtil.SPOUT;
- this.stats = new SpoutExecutorStats(ConfigUtils.samplingRate(stormConf));
+ this.stats = new SpoutExecutorStats(ConfigUtils.samplingRate(stormConf),Utils.getInt(stormConf.get(Config.NUM_STAT_BUCKETS)));
} else if (bolts.containsKey(componentId)) {
this.type = StatsUtil.BOLT;
- this.stats = new BoltExecutorStats(ConfigUtils.samplingRate(stormConf));
+ this.stats = new BoltExecutorStats(ConfigUtils.samplingRate(stormConf),Utils.getInt(stormConf.get(Config.NUM_STAT_BUCKETS)));
} else {
throw new RuntimeException("Could not find " + componentId + " in " + topology);
}
@@ -183,10 +183,10 @@ public abstract class Executor implements Callable, EventHandler<Object> {
String type = getExecutorType(workerTopologyContext, componentId);
if (StatsUtil.SPOUT.equals(type)) {
executor = new SpoutExecutor(workerState, executorId, credentials);
- executor.stats = new SpoutExecutorStats(ConfigUtils.samplingRate(executor.getStormConf()));
+ executor.stats = new SpoutExecutorStats(ConfigUtils.samplingRate(executor.getStormConf()),Utils.getInt(executor.getStormConf().get(Config.NUM_STAT_BUCKETS)));
} else {
executor = new BoltExecutor(workerState, executorId, credentials);
- executor.stats = new BoltExecutorStats(ConfigUtils.samplingRate(executor.getStormConf()));
+ executor.stats = new BoltExecutorStats(ConfigUtils.samplingRate(executor.getStormConf()),Utils.getInt(executor.getStormConf().get(Config.NUM_STAT_BUCKETS)));
}
Map<Integer, Task> idToTask = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/storm/blob/de47d1bb/storm-core/src/jvm/org/apache/storm/stats/BoltExecutorStats.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/stats/BoltExecutorStats.java b/storm-core/src/jvm/org/apache/storm/stats/BoltExecutorStats.java
index 6102549..1bacab2 100644
--- a/storm-core/src/jvm/org/apache/storm/stats/BoltExecutorStats.java
+++ b/storm-core/src/jvm/org/apache/storm/stats/BoltExecutorStats.java
@@ -36,14 +36,14 @@ public class BoltExecutorStats extends CommonStats {
public static final String PROCESS_LATENCIES = "process-latencies";
public static final String EXECUTE_LATENCIES = "execute-latencies";
- public BoltExecutorStats(int rate) {
- super(rate);
-
- this.put(ACKED, new MultiCountStatAndMetric(NUM_STAT_BUCKETS));
- this.put(FAILED, new MultiCountStatAndMetric(NUM_STAT_BUCKETS));
- this.put(EXECUTED, new MultiCountStatAndMetric(NUM_STAT_BUCKETS));
- this.put(PROCESS_LATENCIES, new MultiLatencyStatAndMetric(NUM_STAT_BUCKETS));
- this.put(EXECUTE_LATENCIES, new MultiLatencyStatAndMetric(NUM_STAT_BUCKETS));
+ public BoltExecutorStats(int rate,int numStatBuckets) {
+ super(rate,numStatBuckets);
+
+ this.put(ACKED, new MultiCountStatAndMetric(numStatBuckets));
+ this.put(FAILED, new MultiCountStatAndMetric(numStatBuckets));
+ this.put(EXECUTED, new MultiCountStatAndMetric(numStatBuckets));
+ this.put(PROCESS_LATENCIES, new MultiLatencyStatAndMetric(numStatBuckets));
+ this.put(EXECUTE_LATENCIES, new MultiLatencyStatAndMetric(numStatBuckets));
}
public MultiCountStatAndMetric getAcked() {
http://git-wip-us.apache.org/repos/asf/storm/blob/de47d1bb/storm-core/src/jvm/org/apache/storm/stats/CommonStats.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/stats/CommonStats.java b/storm-core/src/jvm/org/apache/storm/stats/CommonStats.java
index 9b9a4f3..a66a92c 100644
--- a/storm-core/src/jvm/org/apache/storm/stats/CommonStats.java
+++ b/storm-core/src/jvm/org/apache/storm/stats/CommonStats.java
@@ -26,7 +26,6 @@ import org.apache.storm.metric.internal.MultiLatencyStatAndMetric;
@SuppressWarnings("unchecked")
public abstract class CommonStats {
- public static final int NUM_STAT_BUCKETS = 20;
public static final String RATE = "rate";
@@ -37,10 +36,10 @@ public abstract class CommonStats {
protected final int rate;
protected final Map<String, IMetric> metricMap = new HashMap<>();
- public CommonStats(int rate) {
+ public CommonStats(int rate,int numStatBuckets) {
this.rate = rate;
- this.put(EMITTED, new MultiCountStatAndMetric(NUM_STAT_BUCKETS));
- this.put(TRANSFERRED, new MultiCountStatAndMetric(NUM_STAT_BUCKETS));
+ this.put(EMITTED, new MultiCountStatAndMetric(numStatBuckets));
+ this.put(TRANSFERRED, new MultiCountStatAndMetric(numStatBuckets));
}
public int getRate() {
http://git-wip-us.apache.org/repos/asf/storm/blob/de47d1bb/storm-core/src/jvm/org/apache/storm/stats/SpoutExecutorStats.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/stats/SpoutExecutorStats.java b/storm-core/src/jvm/org/apache/storm/stats/SpoutExecutorStats.java
index c7ba844..0addbdc 100644
--- a/storm-core/src/jvm/org/apache/storm/stats/SpoutExecutorStats.java
+++ b/storm-core/src/jvm/org/apache/storm/stats/SpoutExecutorStats.java
@@ -30,11 +30,11 @@ public class SpoutExecutorStats extends CommonStats {
public static final String FAILED = "failed";
public static final String COMPLETE_LATENCIES = "complete-latencies";
- public SpoutExecutorStats(int rate) {
- super(rate);
- this.put(ACKED, new MultiCountStatAndMetric(NUM_STAT_BUCKETS));
- this.put(FAILED, new MultiCountStatAndMetric(NUM_STAT_BUCKETS));
- this.put(COMPLETE_LATENCIES, new MultiLatencyStatAndMetric(NUM_STAT_BUCKETS));
+ public SpoutExecutorStats(int rate,int numStatBuckets) {
+ super(rate,numStatBuckets);
+ this.put(ACKED, new MultiCountStatAndMetric(numStatBuckets));
+ this.put(FAILED, new MultiCountStatAndMetric(numStatBuckets));
+ this.put(COMPLETE_LATENCIES, new MultiLatencyStatAndMetric(numStatBuckets));
}
public MultiCountStatAndMetric getAcked() {
[3/3] storm git commit: STORM-2363: CHANGELOG
Posted by ka...@apache.org.
STORM-2363: CHANGELOG
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c4170ab1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c4170ab1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c4170ab1
Branch: refs/heads/master
Commit: c4170ab16094c16f213704ecf48b0feeb424d19a
Parents: dba325a
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Wed Apr 5 11:05:01 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Wed Apr 5 11:05:01 2017 +0900
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c4170ab1/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a829f6b..9f98ff2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
\ufeff## 2.0.0
+ * STORM-2363: Provide configuration to set the number of RollingWindow
* STORM-2058: Fix Maven warnings about missing reporting.plugins.plugin.version
* STORM-1369: Add MapState implementation to storm-cassandra
* STORM-2432: Storm-Kafka-Client Trident Spout Seeks Incorrect Offset With UNCOMMITTED_LATEST Strategy
[2/3] storm git commit: Merge branch 'STORM-2363-merge'
Posted by ka...@apache.org.
Merge branch 'STORM-2363-merge'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/dba325aa
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/dba325aa
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/dba325aa
Branch: refs/heads/master
Commit: dba325aa72eab65f45078ef79a2a4a619e189ef8
Parents: b2b9ecf de47d1b
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Wed Apr 5 11:04:47 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Wed Apr 5 11:04:47 2017 +0900
----------------------------------------------------------------------
conf/defaults.yaml | 2 ++
storm-core/src/jvm/org/apache/storm/Config.java | 7 +++++++
.../src/jvm/org/apache/storm/executor/Executor.java | 8 ++++----
.../org/apache/storm/stats/BoltExecutorStats.java | 16 ++++++++--------
.../src/jvm/org/apache/storm/stats/CommonStats.java | 7 +++----
.../org/apache/storm/stats/SpoutExecutorStats.java | 10 +++++-----
6 files changed, 29 insertions(+), 21 deletions(-)
----------------------------------------------------------------------