You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2017/07/07 14:17:41 UTC

[1/4] storm git commit: STORM-2610: Fixed the metrics

Repository: storm
Updated Branches:
  refs/heads/master 02598af36 -> d7c781891


STORM-2610: Fixed the metrics


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5a239977
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5a239977
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5a239977

Branch: refs/heads/master
Commit: 5a23997720a0fec60f137222182fa43d848a1e78
Parents: 86e389c
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Jun 30 16:34:01 2017 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Jun 30 16:34:01 2017 -0500

----------------------------------------------------------------------
 .../storm/starter/ThroughputVsLatency.java      |  1 +
 .../daemon/metrics/SpoutThrottlingMetrics.java  | 31 +++++++-------------
 .../storm/executor/spout/SpoutExecutor.java     | 16 +++++-----
 .../org/apache/storm/utils/NimbusClient.java    |  4 +++
 4 files changed, 24 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/5a239977/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
index 1af5844..96c13c5 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ThroughputVsLatency.java
@@ -371,6 +371,7 @@ public class ThroughputVsLatency {
             }
         } finally {
             kill(client.getClient(), name);
+            System.exit(0);
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/5a239977/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java b/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
index 1d2bffe..673d006 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.storm.daemon.metrics;
 
 import org.apache.storm.metric.api.CountMetric;
@@ -26,32 +27,20 @@ public class SpoutThrottlingMetrics extends BuiltinMetrics {
     private final CountMetric skippedInactive = new CountMetric();
 
     public SpoutThrottlingMetrics() {
-        this.metricMap.put("skipped-max-spout", skippedMaxSpout);
-        this.metricMap.put("skipped-throttle", skippedThrottle);
-        this.metricMap.put("skipped-inactive", skippedInactive);
-    }
-
-    public CountMetric getSkippedMaxSpout() {
-        return skippedMaxSpout;
-    }
-
-    public CountMetric getSkippedThrottle() {
-        return skippedThrottle;
-    }
-
-    public CountMetric getSkippedInactive() {
-        return skippedInactive;
+        metricMap.put("skipped-max-spout-ms", skippedMaxSpout);
+        metricMap.put("skipped-throttle-ms", skippedThrottle);
+        metricMap.put("skipped-inactive-ms", skippedInactive);
     }
 
-    public void skippedMaxSpout(CommonStats stats) {
-        this.skippedMaxSpout.incrBy(stats.getRate());
+    public void skippedMaxSpoutMs(long ms) {
+        this.skippedMaxSpout.incrBy(ms);
     }
 
-    public void skippedThrottle(CommonStats stats) {
-        this.skippedThrottle.incrBy(stats.getRate());
+    public void skippedThrottleMs(long ms) {
+        this.skippedThrottle.incrBy(ms);
     }
 
-    public void skippedInactive(CommonStats stats) {
-        this.skippedInactive.incrBy(stats.getRate());
+    public void skippedInactiveMs(long ms) {
+        this.skippedInactive.incrBy(ms);
     }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/5a239977/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java b/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java
index 36f1892..3a03cf7 100644
--- a/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java
+++ b/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java
@@ -139,10 +139,10 @@ public class SpoutExecutor extends Executor {
             public Object call() throws Exception {
                 receiveQueue.consumeBatch(SpoutExecutor.this);
 
-                long currCount = emittedCount.get();
-                boolean throttleOn = backPressureEnabled && SpoutExecutor.this.throttleOn.get();
-                boolean reachedMaxSpoutPending = (maxSpoutPending != 0) && (pending.size() >= maxSpoutPending);
-                boolean isActive = stormActive.get();
+                final long currCount = emittedCount.get();
+                final boolean throttleOn = backPressureEnabled && SpoutExecutor.this.throttleOn.get();
+                final boolean reachedMaxSpoutPending = (maxSpoutPending != 0) && (pending.size() >= maxSpoutPending);
+                final boolean isActive = stormActive.get();
                 if (isActive) {
                     if (!lastActive.get()) {
                         lastActive.set(true);
@@ -164,16 +164,18 @@ public class SpoutExecutor extends Executor {
                             spout.deactivate();
                         }
                     }
+                    long start = Time.currentTimeMillis();
                     Time.sleep(100);
-                    spoutThrottlingMetrics.skippedInactive(stats);
+                    spoutThrottlingMetrics.skippedInactiveMs(Time.currentTimeMillis() - start);
                 }
                 if (currCount == emittedCount.get() && isActive) {
                     emptyEmitStreak.increment();
+                    long start = Time.currentTimeMillis();
                     spoutWaitStrategy.emptyEmit(emptyEmitStreak.get());
                     if (throttleOn) {
-                        spoutThrottlingMetrics.skippedThrottle(stats);
+                        spoutThrottlingMetrics.skippedThrottleMs(Time.currentTimeMillis() - start);
                     } else if (reachedMaxSpoutPending) {
-                        spoutThrottlingMetrics.skippedMaxSpout(stats);
+                        spoutThrottlingMetrics.skippedMaxSpoutMs(Time.currentTimeMillis() - start);
                     }
                 } else {
                     emptyEmitStreak.set(0);

http://git-wip-us.apache.org/repos/asf/storm/blob/5a239977/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java b/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java
index ea13b79..c619400 100644
--- a/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java
+++ b/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java
@@ -96,6 +96,10 @@ public class NimbusClient extends ThriftClient {
         if (override != null) {
             return new NimbusClient(override);
         }
+        Map<String, Object> fullConf = Utils.readStormConfig();
+        fullConf.putAll(Utils.readCommandLineOpts());
+        fullConf.putAll(conf);
+        conf = fullConf;
         if (conf.containsKey(Config.STORM_DO_AS_USER)) {
             if (asUser != null && !asUser.isEmpty()) {
                 LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, config will take precedence."


[2/4] storm git commit: addressed review comments

Posted by bo...@apache.org.
addressed review comments


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a164ceda
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a164ceda
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a164ceda

Branch: refs/heads/master
Commit: a164ceda95d52f8f4a2537ea4f98ba4dbc84b5b7
Parents: 5a23997
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Jul 6 08:07:50 2017 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Jul 6 08:07:50 2017 -0500

----------------------------------------------------------------------
 .../daemon/metrics/SpoutThrottlingMetrics.java   | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/a164ceda/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java b/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
index 673d006..a5f26b8 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/metrics/SpoutThrottlingMetrics.java
@@ -19,28 +19,27 @@
 package org.apache.storm.daemon.metrics;
 
 import org.apache.storm.metric.api.CountMetric;
-import org.apache.storm.stats.CommonStats;
 
 public class SpoutThrottlingMetrics extends BuiltinMetrics {
-    private final CountMetric skippedMaxSpout = new CountMetric();
-    private final CountMetric skippedThrottle = new CountMetric();
-    private final CountMetric skippedInactive = new CountMetric();
+    private final CountMetric skippedMaxSpoutMs = new CountMetric();
+    private final CountMetric skippedThrottleMs = new CountMetric();
+    private final CountMetric skippedInactiveMs = new CountMetric();
 
     public SpoutThrottlingMetrics() {
-        metricMap.put("skipped-max-spout-ms", skippedMaxSpout);
-        metricMap.put("skipped-throttle-ms", skippedThrottle);
-        metricMap.put("skipped-inactive-ms", skippedInactive);
+        metricMap.put("skipped-max-spout-ms", skippedMaxSpoutMs);
+        metricMap.put("skipped-throttle-ms", skippedThrottleMs);
+        metricMap.put("skipped-inactive-ms", skippedInactiveMs);
     }
 
     public void skippedMaxSpoutMs(long ms) {
-        this.skippedMaxSpout.incrBy(ms);
+        this.skippedMaxSpoutMs.incrBy(ms);
     }
 
     public void skippedThrottleMs(long ms) {
-        this.skippedThrottle.incrBy(ms);
+        this.skippedThrottleMs.incrBy(ms);
     }
 
     public void skippedInactiveMs(long ms) {
-        this.skippedInactive.incrBy(ms);
+        this.skippedInactiveMs.incrBy(ms);
     }
 }


[3/4] storm git commit: Merge branch 'STORM-2610' of https://github.com/revans2/incubator-storm into STORM-2610

Posted by bo...@apache.org.
Merge branch 'STORM-2610' of https://github.com/revans2/incubator-storm into STORM-2610

STORM-2610: Fixed the throttle metrics


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e138df8e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e138df8e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e138df8e

Branch: refs/heads/master
Commit: e138df8e46566d1b9b061265399e5e44986d4d1a
Parents: 02598af a164ced
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Fri Jul 7 08:50:07 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Fri Jul 7 08:50:07 2017 -0500

----------------------------------------------------------------------
 .../storm/starter/ThroughputVsLatency.java      |  1 +
 .../daemon/metrics/SpoutThrottlingMetrics.java  | 38 +++++++-------------
 .../storm/executor/spout/SpoutExecutor.java     | 16 +++++----
 .../org/apache/storm/utils/NimbusClient.java    |  4 +++
 4 files changed, 27 insertions(+), 32 deletions(-)
----------------------------------------------------------------------



[4/4] storm git commit: Added STORM-2610 to Changelog

Posted by bo...@apache.org.
Added STORM-2610 to Changelog


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d7c78189
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d7c78189
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d7c78189

Branch: refs/heads/master
Commit: d7c781891fd0cf409335ed1b70397e3c6747475e
Parents: e138df8
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Fri Jul 7 08:50:43 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Fri Jul 7 08:50:43 2017 -0500

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/d7c78189/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 97f7d08..669cfee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 2.0.0
+ * STORM-2610: Fixed throttle metrics
  * STORM-2609: Simple command line DRPC Client
  * STORM-2613: Tuples that aren't sampled shouldn't be considered for execute latency calculations
  * STORM-2421: support lists of childopts in DaemonConfig