You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2016/06/13 05:01:37 UTC
[3/5] storm git commit: STORM-1723 Introduce ClusterMetricsConsumer
STORM-1723 Introduce ClusterMetricsConsumer
* ClusterMetricsConsumer publishes cluster-side related metrics into consumers
* like MetricsConsumer for topology metrics
* Users can implement IClusterMetricsConsumer and configure to cluster conf. file to take effect
* Please refer conf/storm.yaml.example for more details on configuring
* Nimbus should be launched with additional jars which are needed for IClusterMetricsConsumer
* Also did some refactor to nimbus.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/622b6d13
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/622b6d13
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/622b6d13
Branch: refs/heads/1.x-branch
Commit: 622b6d13594a40aa1a171c14ff660375977ea336
Parents: 9c6e6df
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Thu Apr 21 20:40:51 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Sun Jun 12 21:22:09 2016 +0900
----------------------------------------------------------------------
conf/defaults.yaml | 3 +
conf/storm.yaml.example | 9 +
log4j2/cluster.xml | 14 +
.../src/clj/org/apache/storm/daemon/nimbus.clj | 1544 +++++++++---------
storm-core/src/jvm/org/apache/storm/Config.java | 17 +-
.../metric/ClusterMetricsConsumerExecutor.java | 76 +
.../metric/LoggingClusterMetricsConsumer.java | 86 +
.../org/apache/storm/metric/api/DataPoint.java | 43 +
.../metric/api/IClusterMetricsConsumer.java | 63 +
.../storm/metric/api/IMetricsConsumer.java | 2 +
.../storm/validation/ConfigValidation.java | 16 +
11 files changed, 1131 insertions(+), 742 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/622b6d13/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 3f6a1c9..c7a908f 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -285,3 +285,6 @@ pacemaker.kerberos.users: []
#default storm daemon metrics reporter plugins
storm.daemon.metrics.reporter.plugins:
- "org.apache.storm.daemon.metrics.reporters.JmxPreparableReporter"
+
+# configuration of cluster metrics consumer
+storm.cluster.metrics.consumer.publish.interval.secs: 60
http://git-wip-us.apache.org/repos/asf/storm/blob/622b6d13/conf/storm.yaml.example
----------------------------------------------------------------------
diff --git a/conf/storm.yaml.example b/conf/storm.yaml.example
index 7df3e9d..17589f1 100644
--- a/conf/storm.yaml.example
+++ b/conf/storm.yaml.example
@@ -46,3 +46,12 @@
# parallelism.hint: 1
# argument:
# - endpoint: "metrics-collector.mycompany.org"
+
+## Cluster Metrics Consumers
+# storm.cluster.metrics.consumer.register:
+# - class: "org.apache.storm.metric.LoggingClusterMetricsConsumer"
+# - class: "org.mycompany.MyMetricsConsumer"
+# argument:
+# - endpoint: "metrics-collector.mycompany.org"
+#
+# storm.cluster.metrics.consumer.publish.interval.secs: 60
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/622b6d13/log4j2/cluster.xml
----------------------------------------------------------------------
diff --git a/log4j2/cluster.xml b/log4j2/cluster.xml
index eddfae7..e911823 100644
--- a/log4j2/cluster.xml
+++ b/log4j2/cluster.xml
@@ -54,6 +54,17 @@
</Policies>
<DefaultRolloverStrategy max="9"/>
</RollingFile>
+ <RollingFile name="METRICS"
+ fileName="${sys:storm.log.dir}/${sys:logfile.name}.metrics"
+ filePattern="${sys:storm.log.dir}/${sys:logfile.name}.metrics.%i.gz">
+ <PatternLayout>
+ <pattern>${patternMetrics}</pattern>
+ </PatternLayout>
+ <Policies>
+ <SizeBasedTriggeringPolicy size="2 MB"/>
+ </Policies>
+ <DefaultRolloverStrategy max="9"/>
+ </RollingFile>
<Syslog name="syslog" format="RFC5424" charset="UTF-8" host="localhost" port="514"
protocol="UDP" appName="[${sys:daemon.name}]" mdcId="mdc" includeMDC="true"
facility="LOCAL5" enterpriseNumber="18060" newLine="true" exceptionPattern="%rEx{full}"
@@ -69,6 +80,9 @@
<AppenderRef ref="THRIFT-ACCESS"/>
<AppenderRef ref="syslog"/>
</Logger>
+ <Logger name="org.apache.storm.metric.LoggingClusterMetricsConsumer" level="info" additivity="false">
+ <appender-ref ref="METRICS"/>
+ </Logger>
<root level="info"> <!-- We log everything -->
<appender-ref ref="A1"/>
<appender-ref ref="syslog"/>