You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2017/07/07 14:17:41 UTC
[1/4] storm git commit: STORM-2610: Fixed the metrics
Repository: storm
Updated Branches:
refs/heads/master 02598af36 -> d7c781891
STORM-2610: Fixed the metrics
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5a239977
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5a239977
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5a239977
Branch: refs/heads/master
Commit: 5a23997720a0fec60f137222182fa43d848a1e78
Parents: 86e389c
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Jun 30 16:34:01 2017 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Jun 30 16:34:01 2017 -0500
----------------------------------------------------------------------
.../storm/starter/ThroughputVsLatency.java | 1 +
.../daemon/metrics/SpoutThrottlingMetrics.java | 31 +++++++-------------
.../storm/executor/spout/SpoutExecutor.java | 16 +++++-----
.../org/apache/storm/utils/NimbusClient.java | 4 +++
4 files changed, 24 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5a239977/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
index 1af5844..96c13c5 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
@@ -371,6 +371,7 @@ public class ThroughputVsLatency {
}
} finally {
kill(client.getClient(), name);
+ System.exit(0);
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5a239977/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java b/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
index 1d2bffe..673d006 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.metrics;
import org.apache.storm.metric.api.CountMetric;
@@ -26,32 +27,20 @@ public class SpoutThrottlingMetrics extends BuiltinMetrics {
private final CountMetric skippedInactive = new CountMetric();
public SpoutThrottlingMetrics() {
- this.metricMap.put("skipped-max-spout", skippedMaxSpout);
- this.metricMap.put("skipped-throttle", skippedThrottle);
- this.metricMap.put("skipped-inactive", skippedInactive);
- }
-
- public CountMetric getSkippedMaxSpout() {
- return skippedMaxSpout;
- }
-
- public CountMetric getSkippedThrottle() {
- return skippedThrottle;
- }
-
- public CountMetric getSkippedInactive() {
- return skippedInactive;
+ metricMap.put("skipped-max-spout-ms", skippedMaxSpout);
+ metricMap.put("skipped-throttle-ms", skippedThrottle);
+ metricMap.put("skipped-inactive-ms", skippedInactive);
}
- public void skippedMaxSpout(CommonStats stats) {
- this.skippedMaxSpout.incrBy(stats.getRate());
+ public void skippedMaxSpoutMs(long ms) {
+ this.skippedMaxSpout.incrBy(ms);
}
- public void skippedThrottle(CommonStats stats) {
- this.skippedThrottle.incrBy(stats.getRate());
+ public void skippedThrottleMs(long ms) {
+ this.skippedThrottle.incrBy(ms);
}
- public void skippedInactive(CommonStats stats) {
- this.skippedInactive.incrBy(stats.getRate());
+ public void skippedInactiveMs(long ms) {
+ this.skippedInactive.incrBy(ms);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5a239977/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java b/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java
index 36f1892..3a03cf7 100644
--- a/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java
+++ b/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java
@@ -139,10 +139,10 @@ public class SpoutExecutor extends Executor {
public Object call() throws Exception {
receiveQueue.consumeBatch(SpoutExecutor.this);
- long currCount = emittedCount.get();
- boolean throttleOn = backPressureEnabled && SpoutExecutor.this.throttleOn.get();
- boolean reachedMaxSpoutPending = (maxSpoutPending != 0) && (pending.size() >= maxSpoutPending);
- boolean isActive = stormActive.get();
+ final long currCount = emittedCount.get();
+ final boolean throttleOn = backPressureEnabled && SpoutExecutor.this.throttleOn.get();
+ final boolean reachedMaxSpoutPending = (maxSpoutPending != 0) && (pending.size() >= maxSpoutPending);
+ final boolean isActive = stormActive.get();
if (isActive) {
if (!lastActive.get()) {
lastActive.set(true);
@@ -164,16 +164,18 @@ public class SpoutExecutor extends Executor {
spout.deactivate();
}
}
+ long start = Time.currentTimeMillis();
Time.sleep(100);
- spoutThrottlingMetrics.skippedInactive(stats);
+ spoutThrottlingMetrics.skippedInactiveMs(Time.currentTimeMillis() - start);
}
if (currCount == emittedCount.get() && isActive) {
emptyEmitStreak.increment();
+ long start = Time.currentTimeMillis();
spoutWaitStrategy.emptyEmit(emptyEmitStreak.get());
if (throttleOn) {
- spoutThrottlingMetrics.skippedThrottle(stats);
+ spoutThrottlingMetrics.skippedThrottleMs(Time.currentTimeMillis() - start);
} else if (reachedMaxSpoutPending) {
- spoutThrottlingMetrics.skippedMaxSpout(stats);
+ spoutThrottlingMetrics.skippedMaxSpoutMs(Time.currentTimeMillis() - start);
}
} else {
emptyEmitStreak.set(0);
http://git-wip-us.apache.org/repos/asf/storm/blob/5a239977/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java b/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java
index ea13b79..c619400 100644
--- a/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java
+++ b/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java
@@ -96,6 +96,10 @@ public class NimbusClient extends ThriftClient {
if (override != null) {
return new NimbusClient(override);
}
+ Map<String, Object> fullConf = Utils.readStormConfig();
+ fullConf.putAll(Utils.readCommandLineOpts());
+ fullConf.putAll(conf);
+ conf = fullConf;
if (conf.containsKey(Config.STORM_DO_AS_USER)) {
if (asUser != null && !asUser.isEmpty()) {
LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, config will take precedence."
[2/4] storm git commit: addressed review comments
Posted by bo...@apache.org.
addressed review comments
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a164ceda
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a164ceda
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a164ceda
Branch: refs/heads/master
Commit: a164ceda95d52f8f4a2537ea4f98ba4dbc84b5b7
Parents: 5a23997
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Jul 6 08:07:50 2017 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Jul 6 08:07:50 2017 -0500
----------------------------------------------------------------------
.../daemon/metrics/SpoutThrottlingMetrics.java | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a164ceda/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java b/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
index 673d006..a5f26b8 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
@@ -19,28 +19,27 @@
package org.apache.storm.daemon.metrics;
import org.apache.storm.metric.api.CountMetric;
-import org.apache.storm.stats.CommonStats;
public class SpoutThrottlingMetrics extends BuiltinMetrics {
- private final CountMetric skippedMaxSpout = new CountMetric();
- private final CountMetric skippedThrottle = new CountMetric();
- private final CountMetric skippedInactive = new CountMetric();
+ private final CountMetric skippedMaxSpoutMs = new CountMetric();
+ private final CountMetric skippedThrottleMs = new CountMetric();
+ private final CountMetric skippedInactiveMs = new CountMetric();
public SpoutThrottlingMetrics() {
- metricMap.put("skipped-max-spout-ms", skippedMaxSpout);
- metricMap.put("skipped-throttle-ms", skippedThrottle);
- metricMap.put("skipped-inactive-ms", skippedInactive);
+ metricMap.put("skipped-max-spout-ms", skippedMaxSpoutMs);
+ metricMap.put("skipped-throttle-ms", skippedThrottleMs);
+ metricMap.put("skipped-inactive-ms", skippedInactiveMs);
}
public void skippedMaxSpoutMs(long ms) {
- this.skippedMaxSpout.incrBy(ms);
+ this.skippedMaxSpoutMs.incrBy(ms);
}
public void skippedThrottleMs(long ms) {
- this.skippedThrottle.incrBy(ms);
+ this.skippedThrottleMs.incrBy(ms);
}
public void skippedInactiveMs(long ms) {
- this.skippedInactive.incrBy(ms);
+ this.skippedInactiveMs.incrBy(ms);
}
}
[3/4] storm git commit: Merge branch 'STORM-2610' of
https://github.com/revans2/incubator-storm into STORM-2610
Posted by bo...@apache.org.
Merge branch 'STORM-2610' of https://github.com/revans2/incubator-storm into STORM-2610
STORM-2610: Fixed the throttle metrics
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e138df8e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e138df8e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e138df8e
Branch: refs/heads/master
Commit: e138df8e46566d1b9b061265399e5e44986d4d1a
Parents: 02598af a164ced
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Fri Jul 7 08:50:07 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Fri Jul 7 08:50:07 2017 -0500
----------------------------------------------------------------------
.../storm/starter/ThroughputVsLatency.java | 1 +
.../daemon/metrics/SpoutThrottlingMetrics.java | 38 +++++++-------------
.../storm/executor/spout/SpoutExecutor.java | 16 +++++----
.../org/apache/storm/utils/NimbusClient.java | 4 +++
4 files changed, 27 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
[4/4] storm git commit: Added STORM-2610 to Changelog
Posted by bo...@apache.org.
Added STORM-2610 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d7c78189
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d7c78189
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d7c78189
Branch: refs/heads/master
Commit: d7c781891fd0cf409335ed1b70397e3c6747475e
Parents: e138df8
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Fri Jul 7 08:50:43 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Fri Jul 7 08:50:43 2017 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d7c78189/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 97f7d08..669cfee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-2610: Fixed throttle metrics
* STORM-2609: Simple command line DRPC Client
* STORM-2613: Tuples that aren't sampled shouldn't be considered for execute latency calculations
* STORM-2421: support lists of childopts in DaemonConfig