You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/07/16 16:48:11 UTC
[06/11] git commit: seperate out argument parsing in Monitor class;
remove 'waiting' status in mertics; add argument -s stream to specify stream
seperate out argument parsing in Monitor class; remove 'waiting' status in mertics; add argument -s stream to specify stream
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/81cf56ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/81cf56ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/81cf56ed
Branch: refs/heads/master
Commit: 81cf56edd053ae6e392a81577e0c4cb7a305cc36
Parents: edcaf71
Author: jiahong.ljh <ji...@alibaba-inc.com>
Authored: Mon Jun 30 18:06:38 2014 +0800
Committer: jiahong.ljh <ji...@alibaba-inc.com>
Committed: Mon Jun 30 18:06:38 2014 +0800
----------------------------------------------------------------------
bin/storm | 7 +-
pom.xml | 6 -
storm-core/pom.xml | 4 -
.../src/clj/backtype/storm/command/monitor.clj | 38 ++++
.../src/jvm/backtype/storm/command/Monitor.java | 197 -------------------
.../src/jvm/backtype/storm/utils/Monitor.java | 178 +++++++++++++++++
6 files changed, 220 insertions(+), 210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/bin/storm
----------------------------------------------------------------------
diff --git a/bin/storm b/bin/storm
index 0600d45..9ec4ef4 100755
--- a/bin/storm
+++ b/bin/storm
@@ -391,12 +391,13 @@ def print_classpath():
print get_classpath([])
def monitor(*args):
- """Syntax: [storm monitor]
+ """Syntax: [storm monitor [-i interval-secs] -t topology-name -m component-id [-s stream-id] [-w [emitted | transferred]]*]
- Monitor given topologies' given components' or type of components' throughput interactively.
+ Monitor topology's throughput interactively.
+ One can specify poll-interval, topology's name, component-id, stream-id, watch-item[emitted | transferred]
"""
exec_storm_class(
- "backtype.storm.command.Monitor",
+ "backtype.storm.command.monitor",
args=args,
jvmtype="-client",
extrajars=[USER_CONF_DIR, STORM_DIR + "/bin"])
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 484f5f4..aa93fcc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -464,12 +464,6 @@
<version>3.8.1</version>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>args4j</groupId>
- <artifactId>args4j</artifactId>
- <version>2.0.16</version>
- </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 33f6cc6..fec6218 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -165,10 +165,6 @@
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</dependency>
- <dependency>
- <groupId>args4j</groupId>
- <artifactId>args4j</artifactId>
- </dependency>
<dependency>
<groupId>org.mockito</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/storm-core/src/clj/backtype/storm/command/monitor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/command/monitor.clj b/storm-core/src/clj/backtype/storm/command/monitor.clj
new file mode 100644
index 0000000..0869032
--- /dev/null
+++ b/storm-core/src/clj/backtype/storm/command/monitor.clj
@@ -0,0 +1,38 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+(ns backtype.storm.command.monitor
+ (:use [clojure.tools.cli :only [cli]])
+ (:use [backtype.storm thrift config log])
+ (:import [backtype.storm.utils Monitor])
+ (:gen-class)
+ )
+
+(defn -main [& args]
+ (let [[{interval :interval topology :topology component :component stream :stream watch :watch} [_] _]
+ (cli args ["-i" "--interval" :default 4 :parse-fn #(Integer/parseInt %)]
+ ["-t" "--topology" :default nil]
+ ["-m" "--component" :default nil]
+ ["-s" "--stream" :default "default"]
+ ["-w" "--watch" :default "emitted"])
+ mon (Monitor.)]
+ (if interval (.set_interval mon interval))
+ (if topology (.set_topology mon topology))
+ (if component (.set_component mon component))
+ (if stream (.set_stream mon stream))
+ (if watch (.set_watch mon watch))
+ (with-configured-nimbus-connection nimbus
+ (.metrics mon nimbus)
+ )))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/storm-core/src/jvm/backtype/storm/command/Monitor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/command/Monitor.java b/storm-core/src/jvm/backtype/storm/command/Monitor.java
deleted file mode 100644
index 8f39d0c..0000000
--- a/storm-core/src/jvm/backtype/storm/command/Monitor.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package backtype.storm.command;
-
-import backtype.storm.generated.*;
-import backtype.storm.utils.NimbusClient;
-import backtype.storm.utils.Utils;
-import org.kohsuke.args4j.CmdLineException;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
-import java.util.Map;
-
-public class Monitor {
- @Option(name="--help", aliases={"-h"}, usage="print help message")
- private boolean _help = false;
-
- @Option(name="--interval", aliases={"-i"}, usage="poll frequency in seconds")
- private int _interval = 4;
-
- @Option(name="--name", aliases={"--topologyName"}, metaVar="NAME",
- usage="base name of the topology (numbers may be appended to the end)")
- private String _name;
-
- @Option(name="--component", aliases={"--componentName"}, metaVar="NAME",
- usage="component name of the topology")
- private String _component;
-
- @Option(name="--stat", aliases={"--statItem"}, metaVar="ITEM",
- usage="stat item [emitted | transferred]")
- private String _stat = "emitted";
-
- private static class MetricsState {
- long lastStatted = 0;
- long lastTime = 0;
- }
-
- public void metrics(Nimbus.Client client, int poll, String name, String component, String stat) throws Exception {
- System.out.println("status\ttopologie\tslots\tcomponent\texecutors\texecutorsWithMetrics\ttime-diff ms\t" + stat + "\tthroughput (Kt/s)");
- MetricsState state = new MetricsState();
- long pollMs = poll * 1000;
- long now = System.currentTimeMillis();
- state.lastTime = now;
- long startTime = now;
- long cycle, sleepTime, wakeupTime;
-
- while (metrics(client, name, component, stat, now, state, "WAITING")) {
- now = System.currentTimeMillis();
- cycle = (now - startTime)/pollMs;
- wakeupTime = startTime + (pollMs * (cycle + 1));
- sleepTime = wakeupTime - now;
- if (sleepTime > 0) {
- Thread.sleep(sleepTime);
- }
- now = System.currentTimeMillis();
- }
-
- now = System.currentTimeMillis();
- cycle = (now - startTime)/pollMs;
- wakeupTime = startTime + (pollMs * (cycle + 1));
- sleepTime = wakeupTime - now;
- if (sleepTime > 0) {
- Thread.sleep(sleepTime);
- }
- now = System.currentTimeMillis();
- do {
- metrics(client, name, component, stat, now, state, "RUNNING");
- now = System.currentTimeMillis();
- cycle = (now - startTime)/pollMs;
- wakeupTime = startTime + (pollMs * (cycle + 1));
- sleepTime = wakeupTime - now;
- if (sleepTime > 0) {
- Thread.sleep(sleepTime);
- }
- now = System.currentTimeMillis();
- } while (true);
- }
-
- public boolean metrics(Nimbus.Client client, String name, String component, String stat, long now, MetricsState state, String message) throws Exception {
- long totalStatted = 0;
-
- boolean topologyFound = false;
- boolean componentFound = false;
- int slotsUsed = 0;
- int executors = 0;
- int executorsWithMetrics = 0;
- ClusterSummary summary = client.getClusterInfo();
- for (TopologySummary ts: summary.get_topologies()) {
- if (name.equals(ts.get_name())) {
- topologyFound = true;
- slotsUsed = ts.get_num_workers();
- String id = ts.get_id();
- TopologyInfo info = client.getTopologyInfo(id);
- for (ExecutorSummary es: info.get_executors()) {
- if (component.equals(es.get_component_id())) {
- componentFound = true;
- executors ++;
- ExecutorStats stats = es.get_stats();
- if (stats != null) {
- Map<String,Map<String,Long>> statted =
- "emitted".equals(stat) ? stats.get_emitted() : stats.get_transferred();
- if ( statted != null) {
- Map<String, Long> e2 = statted.get(":all-time");
- if (e2 != null) {
- executorsWithMetrics++;
- //topology messages are always on the default stream, so just count those
- Long dflt = e2.get("default");
- if (dflt != null){
- totalStatted += dflt;
- }
- }
- }
- }
- }
-
-
- }
- }
- }
-
- if (!topologyFound) {
- throw new IllegalArgumentException("topology: " + name + " not found");
- }
- if (!componentFound) {
- throw new IllegalArgumentException("component: " + component + " not fouond");
- }
- long timeDelta = now - state.lastTime;
- long stattedDelta = totalStatted - state.lastStatted;
- state.lastTime = now;
- state.lastStatted = totalStatted;
- double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double)stattedDelta/(double)timeDelta);
- System.out.println(message+"\t"+name+"\t"+slotsUsed+"\t"+component+"\t"+executors+"\t"+executorsWithMetrics+"\t"+timeDelta+"\t"+stattedDelta+"\t"+throughput);
-
- return !(executors > 0 && executorsWithMetrics >= executors);
- }
-
- public void realMain(String[] args) {
- /*** parse command line ***/
- CmdLineParser parser = new CmdLineParser(this);
- parser.setUsageWidth(80);
- try {
- parser.parseArgument(args);
- } catch( CmdLineException e ) {
- System.err.println(e.getMessage());
- _help = true;
- }
- if(_help) {
- parser.printUsage(System.err);
- System.err.println();
- return;
- }
-
- if (_name == null || _name.isEmpty()) {
- throw new IllegalArgumentException("topology name must be something");
- }
-
- if (_component == null || _component.isEmpty()) {
- throw new IllegalArgumentException("component name must be something");
- }
-
- if (_interval <= 0) {
- throw new IllegalArgumentException("poll interval must be positive");
- }
-
- if ( !"transferred".equals(_stat) && !"emitted".equals(_stat)) {
- throw new IllegalArgumentException("stat item must either be transferred or emitted");
- }
-
- Map clusterConf = Utils.readStormConfig();
- clusterConf.putAll(Utils.readCommandLineOpts());
- Nimbus.Client client = NimbusClient.getConfiguredClient(clusterConf).getClient();
- try {
- metrics(client, _interval, _name, _component, _stat);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- public static void main(String[] args) {
- new Monitor().realMain(args);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/81cf56ed/storm-core/src/jvm/backtype/storm/utils/Monitor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Monitor.java b/storm-core/src/jvm/backtype/storm/utils/Monitor.java
new file mode 100644
index 0000000..c517068
--- /dev/null
+++ b/storm-core/src/jvm/backtype/storm/utils/Monitor.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.utils;
+
+import backtype.storm.generated.*;
+import java.util.Map;
+
+public class Monitor {
+ private static final String WATCH_TRANSFERRED = "transferred";
+ private static final String WATCH_EMITTED = "emitted";
+
+ private int _interval = 4;
+ private String _topology;
+ private String _component;
+ private String _stream;
+ private String _watch;
+
+ private static class MetricsState {
+ long lastStatted = 0;
+ long lastTime = 0;
+ }
+
+ private static class Poller {
+ long startTime = 0;
+ long pollMs = 0;
+
+ public long nextPoll() {
+ long now = System.currentTimeMillis();
+ long cycle = (now - startTime) / pollMs;
+ long wakeupTime = startTime + (pollMs * (cycle + 1));
+ long sleepTime = wakeupTime - now;
+ if (sleepTime > 0) {
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ now = System.currentTimeMillis();
+ return now;
+ }
+ }
+
+ public void metrics(Nimbus.Client client) throws Exception {
+ if (_interval <= 0) {
+ throw new IllegalArgumentException("poll interval must be positive");
+ }
+
+ if (_topology == null || _topology.isEmpty()) {
+ throw new IllegalArgumentException("topology name must be something");
+ }
+
+ if (_component == null || _component.isEmpty()) {
+ throw new IllegalArgumentException("component name must be something");
+ }
+
+ if (_stream == null || _stream.isEmpty()) {
+ throw new IllegalArgumentException("stream name must be something");
+ }
+
+ if ( !WATCH_TRANSFERRED.equals(_watch) && !WATCH_EMITTED.equals(_watch)) {
+ throw new IllegalArgumentException("watch item must either be transferred or emitted");
+ }
+ System.out.println("topology\tslots\texecutors\texecutorsWithMetrics\tcomponent\tstream\ttime-diff ms\t" + _watch + "\tthroughput (Kt/s)");
+
+ long pollMs = _interval * 1000;
+
+ MetricsState state = new MetricsState();
+ Poller poller = new Poller();
+ long now = System.currentTimeMillis();
+ state.lastTime = now;
+ state.lastStatted = 0;
+ poller.startTime = now;
+ poller.pollMs = pollMs;
+
+ do {
+ metrics(client, now, state);
+ now = poller.nextPoll();
+ } while (true);
+ }
+
+ public void metrics(Nimbus.Client client, long now, MetricsState state) throws Exception {
+ long totalStatted = 0;
+
+ boolean topologyFound = false;
+ boolean componentFound = false;
+ boolean streamFound = false;
+ int slotsUsed = 0;
+ int executors = 0;
+ int executorsWithMetrics = 0;
+ ClusterSummary summary = client.getClusterInfo();
+ for (TopologySummary ts: summary.get_topologies()) {
+ if (_topology.equals(ts.get_name())) {
+ topologyFound = true;
+ slotsUsed = ts.get_num_workers();
+ String id = ts.get_id();
+ TopologyInfo info = client.getTopologyInfo(id);
+ for (ExecutorSummary es: info.get_executors()) {
+ if (_component.equals(es.get_component_id())) {
+ componentFound = true;
+ executors ++;
+ ExecutorStats stats = es.get_stats();
+ if (stats != null) {
+ Map<String,Map<String,Long>> statted =
+ WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred();
+ if ( statted != null) {
+ Map<String, Long> e2 = statted.get(":all-time");
+ if (e2 != null) {
+ Long stream = e2.get(_stream);
+ if (stream != null){
+ streamFound = true;
+ executorsWithMetrics ++;
+ totalStatted += stream;
+ }
+ }
+ }
+ }
+ }
+
+
+ }
+ }
+ }
+
+ if (!topologyFound) {
+ throw new IllegalArgumentException("topology: " + _topology + " not found");
+ }
+
+ if (!componentFound) {
+ throw new IllegalArgumentException("component: " + _component + " not fouond");
+ }
+
+ if (!streamFound) {
+ throw new IllegalArgumentException("stream: " + _stream + " not fouond");
+ }
+ long timeDelta = now - state.lastTime;
+ long stattedDelta = totalStatted - state.lastStatted;
+ state.lastTime = now;
+ state.lastStatted = totalStatted;
+ double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double)stattedDelta/(double)timeDelta);
+ System.out.println(_topology+"\t"+slotsUsed+"\t"+executors+"\t"+executorsWithMetrics+"\t"+_component+"\t"+_stream+"\t"+timeDelta+"\t"+stattedDelta+"\t"+throughput);
+ }
+
+ public void set_interval(int _interval) {
+ this._interval = _interval;
+ }
+
+ public void set_topology(String _topology) {
+ this._topology = _topology;
+ }
+
+ public void set_component(String _component) {
+ this._component = _component;
+ }
+
+ public void set_stream(String _stream) {
+ this._stream = _stream;
+ }
+
+ public void set_watch(String _watch) {
+ this._watch = _watch;
+ }
+}