You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/07/16 16:48:11 UTC

[06/11] git commit: seperate out argument parsing in Monitor class; remove 'waiting' status in mertics; add argument -s stream to specify stream

seperate out argument parsing in Monitor class; remove 'waiting' status in mertics; add argument -s stream to specify stream


Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/81cf56ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/81cf56ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/81cf56ed

Branch: refs/heads/master
Commit: 81cf56edd053ae6e392a81577e0c4cb7a305cc36
Parents: edcaf71
Author: jiahong.ljh <ji...@alibaba-inc.com>
Authored: Mon Jun 30 18:06:38 2014 +0800
Committer: jiahong.ljh <ji...@alibaba-inc.com>
Committed: Mon Jun 30 18:06:38 2014 +0800

----------------------------------------------------------------------
 bin/storm                                       |   7 +-
 pom.xml                                         |   6 -
 storm-core/pom.xml                              |   4 -
 .../src/clj/backtype/storm/command/monitor.clj  |  38 ++++
 .../src/jvm/backtype/storm/command/Monitor.java | 197 -------------------
 .../src/jvm/backtype/storm/utils/Monitor.java   | 178 +++++++++++++++++
 6 files changed, 220 insertions(+), 210 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/bin/storm
----------------------------------------------------------------------
diff --git a/bin/storm b/bin/storm
index 0600d45..9ec4ef4 100755
--- a/bin/storm
+++ b/bin/storm
@@ -391,12 +391,13 @@ def print_classpath():
     print get_classpath([])
 
 def monitor(*args):
-    """Syntax: [storm monitor]
+    """Syntax: [storm monitor [-i interval-secs] -t topology-name -m component-id [-s stream-id] [-w [emitted | transferred]]*]
 
-    Monitor given topologies' given components' or type of components' throughput interactively.
+    Monitor topology's throughput interactively.
+    One can specify poll-interval, topology's name, component-id, stream-id, watch-item[emitted | transferred]
     """
     exec_storm_class(
-        "backtype.storm.command.Monitor",
+        "backtype.storm.command.monitor",
         args=args,
         jvmtype="-client",
         extrajars=[USER_CONF_DIR, STORM_DIR + "/bin"])

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 484f5f4..aa93fcc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -464,12 +464,6 @@
 		      <version>3.8.1</version>
 		      <scope>test</scope>
 		    </dependency>
-
-            <dependency>
-                <groupId>args4j</groupId>
-                <artifactId>args4j</artifactId>
-                <version>2.0.16</version>
-            </dependency>
         </dependencies>
     </dependencyManagement>
 

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 33f6cc6..fec6218 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -165,10 +165,6 @@
             <groupId>io.netty</groupId>
             <artifactId>netty</artifactId>
         </dependency>
-        <dependency>
-            <groupId>args4j</groupId>
-            <artifactId>args4j</artifactId>
-        </dependency>
 
         <dependency>
             <groupId>org.mockito</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/storm-core/src/clj/backtype/storm/command/monitor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/command/monitor.clj b/storm-core/src/clj/backtype/storm/command/monitor.clj
new file mode 100644
index 0000000..0869032
--- /dev/null
+++ b/storm-core/src/clj/backtype/storm/command/monitor.clj
@@ -0,0 +1,38 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements.  See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership.  The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License.  You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+(ns backtype.storm.command.monitor
+  (:use [clojure.tools.cli :only [cli]])
+  (:use [backtype.storm thrift config log])
+  (:import [backtype.storm.utils Monitor])
+  (:gen-class)
+ )
+
+(defn -main [& args]
+  (let [[{interval :interval topology :topology component :component stream :stream watch :watch} [_] _]
+        (cli args ["-i" "--interval" :default 4 :parse-fn #(Integer/parseInt %)]
+          ["-t" "--topology" :default nil]
+          ["-m" "--component" :default nil]
+          ["-s" "--stream" :default "default"]
+          ["-w" "--watch" :default "emitted"])
+        mon (Monitor.)]
+    (if interval (.set_interval mon interval))
+    (if topology (.set_topology mon topology))
+    (if component (.set_component mon component))
+    (if stream (.set_stream mon stream))
+    (if watch (.set_watch mon watch))
+    (with-configured-nimbus-connection nimbus
+      (.metrics mon nimbus)
+      )))

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/storm-core/src/jvm/backtype/storm/command/Monitor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/command/Monitor.java b/storm-core/src/jvm/backtype/storm/command/Monitor.java
deleted file mode 100644
index 8f39d0c..0000000
--- a/storm-core/src/jvm/backtype/storm/command/Monitor.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package backtype.storm.command;
-
-import backtype.storm.generated.*;
-import backtype.storm.utils.NimbusClient;
-import backtype.storm.utils.Utils;
-import org.kohsuke.args4j.CmdLineException;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
-import java.util.Map;
-
-public class Monitor {
-    @Option(name="--help", aliases={"-h"}, usage="print help message")
-    private boolean _help = false;
-
-    @Option(name="--interval", aliases={"-i"}, usage="poll frequency in seconds")
-    private int _interval = 4;
-
-    @Option(name="--name", aliases={"--topologyName"}, metaVar="NAME",
-            usage="base name of the topology (numbers may be appended to the end)")
-    private String _name;
-
-    @Option(name="--component", aliases={"--componentName"}, metaVar="NAME",
-            usage="component name of the topology")
-    private String _component;
-
-    @Option(name="--stat", aliases={"--statItem"}, metaVar="ITEM",
-            usage="stat item [emitted | transferred]")
-    private String _stat = "emitted";
-
-    private static class MetricsState {
-        long lastStatted = 0;
-        long lastTime = 0;
-    }
-
-    public void metrics(Nimbus.Client client, int poll, String name, String component, String stat) throws Exception {
-        System.out.println("status\ttopologie\tslots\tcomponent\texecutors\texecutorsWithMetrics\ttime-diff ms\t" + stat + "\tthroughput (Kt/s)");
-        MetricsState state = new MetricsState();
-        long pollMs = poll * 1000;
-        long now = System.currentTimeMillis();
-        state.lastTime = now;
-        long startTime = now;
-        long cycle, sleepTime, wakeupTime;
-
-        while (metrics(client, name, component, stat, now, state, "WAITING")) {
-            now = System.currentTimeMillis();
-            cycle = (now - startTime)/pollMs;
-            wakeupTime = startTime + (pollMs * (cycle + 1));
-            sleepTime = wakeupTime - now;
-            if (sleepTime > 0) {
-                Thread.sleep(sleepTime);
-            }
-            now = System.currentTimeMillis();
-        }
-
-        now = System.currentTimeMillis();
-        cycle = (now - startTime)/pollMs;
-        wakeupTime = startTime + (pollMs * (cycle + 1));
-        sleepTime = wakeupTime - now;
-        if (sleepTime > 0) {
-            Thread.sleep(sleepTime);
-        }
-        now = System.currentTimeMillis();
-        do {
-            metrics(client, name, component, stat, now, state, "RUNNING");
-            now = System.currentTimeMillis();
-            cycle = (now - startTime)/pollMs;
-            wakeupTime = startTime + (pollMs * (cycle + 1));
-            sleepTime = wakeupTime - now;
-            if (sleepTime > 0) {
-                Thread.sleep(sleepTime);
-            }
-            now = System.currentTimeMillis();
-        } while (true);
-    }
-
-    public boolean metrics(Nimbus.Client client, String name, String component, String stat, long now, MetricsState state, String message) throws Exception {
-        long totalStatted = 0;
-
-        boolean topologyFound = false;
-        boolean componentFound = false;
-        int slotsUsed = 0;
-        int executors = 0;
-        int executorsWithMetrics = 0;
-        ClusterSummary summary = client.getClusterInfo();
-        for (TopologySummary ts: summary.get_topologies()) {
-            if (name.equals(ts.get_name())) {
-                topologyFound = true;
-                slotsUsed = ts.get_num_workers();
-                String id = ts.get_id();
-                TopologyInfo info = client.getTopologyInfo(id);
-                for (ExecutorSummary es: info.get_executors()) {
-                    if (component.equals(es.get_component_id())) {
-                        componentFound = true;
-                        executors ++;
-                        ExecutorStats stats = es.get_stats();
-                        if (stats != null) {
-                            Map<String,Map<String,Long>> statted =
-                                    "emitted".equals(stat) ? stats.get_emitted() : stats.get_transferred();
-                            if ( statted != null) {
-                                Map<String, Long> e2 = statted.get(":all-time");
-                                if (e2 != null) {
-                                    executorsWithMetrics++;
-                                    //topology messages are always on the default stream, so just count those
-                                    Long dflt = e2.get("default");
-                                    if (dflt != null){
-                                        totalStatted += dflt;
-                                    }
-                                }
-                            }
-                        }
-                    }
-
-
-                }
-            }
-        }
-
-        if (!topologyFound) {
-            throw new IllegalArgumentException("topology: " + name + " not found");
-        }
-        if (!componentFound) {
-            throw new IllegalArgumentException("component: " + component + " not fouond");
-        }
-        long timeDelta = now - state.lastTime;
-        long stattedDelta = totalStatted - state.lastStatted;
-        state.lastTime = now;
-        state.lastStatted = totalStatted;
-        double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double)stattedDelta/(double)timeDelta);
-        System.out.println(message+"\t"+name+"\t"+slotsUsed+"\t"+component+"\t"+executors+"\t"+executorsWithMetrics+"\t"+timeDelta+"\t"+stattedDelta+"\t"+throughput);
-
-        return !(executors > 0 && executorsWithMetrics >= executors);
-    }
-
-    public void realMain(String[] args) {
-        /*** parse command line ***/
-        CmdLineParser parser = new CmdLineParser(this);
-        parser.setUsageWidth(80);
-        try {
-            parser.parseArgument(args);
-        } catch( CmdLineException e ) {
-            System.err.println(e.getMessage());
-            _help = true;
-        }
-        if(_help) {
-            parser.printUsage(System.err);
-            System.err.println();
-            return;
-        }
-
-        if (_name == null || _name.isEmpty()) {
-            throw new IllegalArgumentException("topology name must be something");
-        }
-
-        if (_component == null || _component.isEmpty()) {
-            throw new IllegalArgumentException("component name must be something");
-        }
-
-        if (_interval <= 0) {
-            throw new IllegalArgumentException("poll interval must be positive");
-        }
-
-        if ( !"transferred".equals(_stat) && !"emitted".equals(_stat)) {
-            throw new IllegalArgumentException("stat item must either be transferred or emitted");
-        }
-
-        Map clusterConf = Utils.readStormConfig();
-        clusterConf.putAll(Utils.readCommandLineOpts());
-        Nimbus.Client client = NimbusClient.getConfiguredClient(clusterConf).getClient();
-        try {
-            metrics(client, _interval, _name, _component, _stat);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    public static void main(String[] args) {
-        new Monitor().realMain(args);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/storm-core/src/jvm/backtype/storm/utils/Monitor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Monitor.java b/storm-core/src/jvm/backtype/storm/utils/Monitor.java
new file mode 100644
index 0000000..c517068
--- /dev/null
+++ b/storm-core/src/jvm/backtype/storm/utils/Monitor.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.utils;
+
+import backtype.storm.generated.*;
+import java.util.Map;
+
+public class Monitor {
+    private static final String WATCH_TRANSFERRED = "transferred";
+    private static final String WATCH_EMITTED = "emitted";
+
+    private int _interval = 4;
+    private String _topology;
+    private String _component;
+    private String _stream;
+    private String _watch;
+
+    private static class MetricsState {
+        long lastStatted = 0;
+        long lastTime = 0;
+    }
+
+    private static class Poller {
+        long startTime = 0;
+        long pollMs = 0;
+
+        public long nextPoll() {
+            long now = System.currentTimeMillis();
+            long cycle = (now - startTime) / pollMs;
+            long wakeupTime = startTime + (pollMs * (cycle + 1));
+            long sleepTime = wakeupTime - now;
+            if (sleepTime > 0) {
+                try {
+                    Thread.sleep(sleepTime);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+            now = System.currentTimeMillis();
+            return now;
+        }
+    }
+
+    public void metrics(Nimbus.Client client) throws Exception {
+        if (_interval <= 0) {
+            throw new IllegalArgumentException("poll interval must be positive");
+        }
+
+        if (_topology == null || _topology.isEmpty()) {
+            throw new IllegalArgumentException("topology name must be something");
+        }
+
+        if (_component == null || _component.isEmpty()) {
+            throw new IllegalArgumentException("component name must be something");
+        }
+
+        if (_stream == null || _stream.isEmpty()) {
+            throw new IllegalArgumentException("stream name must be something");
+        }
+
+        if ( !WATCH_TRANSFERRED.equals(_watch) && !WATCH_EMITTED.equals(_watch)) {
+            throw new IllegalArgumentException("watch item must either be transferred or emitted");
+        }
+        System.out.println("topology\tslots\texecutors\texecutorsWithMetrics\tcomponent\tstream\ttime-diff ms\t" + _watch + "\tthroughput (Kt/s)");
+
+        long pollMs = _interval * 1000;
+
+        MetricsState state = new MetricsState();
+        Poller poller = new Poller();
+        long now = System.currentTimeMillis();
+        state.lastTime = now;
+        state.lastStatted = 0;
+        poller.startTime = now;
+        poller.pollMs = pollMs;
+
+        do {
+            metrics(client, now, state);
+            now = poller.nextPoll();
+        } while (true);
+    }
+
+    public void metrics(Nimbus.Client client, long now, MetricsState state) throws Exception {
+        long totalStatted = 0;
+
+        boolean topologyFound = false;
+        boolean componentFound = false;
+        boolean streamFound = false;
+        int slotsUsed = 0;
+        int executors = 0;
+        int executorsWithMetrics = 0;
+        ClusterSummary summary = client.getClusterInfo();
+        for (TopologySummary ts: summary.get_topologies()) {
+            if (_topology.equals(ts.get_name())) {
+                topologyFound = true;
+                slotsUsed = ts.get_num_workers();
+                String id = ts.get_id();
+                TopologyInfo info = client.getTopologyInfo(id);
+                for (ExecutorSummary es: info.get_executors()) {
+                    if (_component.equals(es.get_component_id())) {
+                        componentFound = true;
+                        executors ++;
+                        ExecutorStats stats = es.get_stats();
+                        if (stats != null) {
+                            Map<String,Map<String,Long>> statted =
+                                    WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred();
+                            if ( statted != null) {
+                                Map<String, Long> e2 = statted.get(":all-time");
+                                if (e2 != null) {
+                                    Long stream = e2.get(_stream);
+                                    if (stream != null){
+                                        streamFound = true;
+                                        executorsWithMetrics ++;
+                                        totalStatted += stream;
+                                    }
+                                }
+                            }
+                        }
+                    }
+
+
+                }
+            }
+        }
+
+        if (!topologyFound) {
+            throw new IllegalArgumentException("topology: " + _topology + " not found");
+        }
+
+        if (!componentFound) {
+            throw new IllegalArgumentException("component: " + _component + " not fouond");
+        }
+
+        if (!streamFound) {
+            throw new IllegalArgumentException("stream: " + _stream + " not fouond");
+        }
+        long timeDelta = now - state.lastTime;
+        long stattedDelta = totalStatted - state.lastStatted;
+        state.lastTime = now;
+        state.lastStatted = totalStatted;
+        double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double)stattedDelta/(double)timeDelta);
+        System.out.println(_topology+"\t"+slotsUsed+"\t"+executors+"\t"+executorsWithMetrics+"\t"+_component+"\t"+_stream+"\t"+timeDelta+"\t"+stattedDelta+"\t"+throughput);
+    }
+
+    public void set_interval(int _interval) {
+        this._interval = _interval;
+    }
+
+    public void set_topology(String _topology) {
+        this._topology = _topology;
+    }
+
+    public void set_component(String _component) {
+        this._component = _component;
+    }
+
+    public void set_stream(String _stream) {
+        this._stream = _stream;
+    }
+
+    public void set_watch(String _watch) {
+        this._watch = _watch;
+    }
+}