You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by si...@apache.org on 2013/11/09 19:49:55 UTC
svn commit: r1540366 - in /zookeeper/bookkeeper/trunk: ./
bookkeeper-stats-providers/
bookkeeper-stats-providers/codahale-metrics-provider/
bookkeeper-stats-providers/codahale-metrics-provider/src/
bookkeeper-stats-providers/codahale-metrics-provider/s...
Author: sijie
Date: Sat Nov 9 18:49:55 2013
New Revision: 1540366
URL: http://svn.apache.org/r1540366
Log:
BOOKKEEPER-699: Codahale metrics implementation of stats API (ivank via sijie)
Added:
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1540366&r1=1540365&r2=1540366&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Sat Nov 9 18:49:55 2013
@@ -190,6 +190,8 @@ Trunk (unreleased changes)
BOOKKEEPER-698: Bookie client closure is not considering timeoutExecutor (rakesh via sijie)
+ BOOKKEEPER-699: Codahale metrics implementation of stats API (ivank via sijie)
+
NEW FEATURE:
BOOKKEEPER-562: Ability to tell if a ledger is closed or not (fpj)
Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml?rev=1540366&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml Sat Nov 9 18:49:55 2013
@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>bookkeeper</artifactId>
+ <groupId>org.apache.bookkeeper</groupId>
+ <version>4.3.0-SNAPSHOT</version>
+ <relativePath>../..</relativePath>
+ </parent>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <artifactId>codahale-metrics-provider</artifactId>
+ <name>Stats provider for codahale metrics</name>
+ <url>http://maven.apache.org</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.libdir>${basedir}/lib</project.libdir>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <artifactId>bookkeeper-stats-api</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-graphite</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>13.0.1</version>
+ </dependency>
+ </dependencies>
+</project>
Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java?rev=1540366&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java Sat Nov 9 18:49:55 2013
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import java.net.InetSocketAddress;
+
+import com.codahale.metrics.CsvReporter;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Slf4jReporter;
+import com.google.common.base.Strings;
+import com.google.common.net.HostAndPort;
+import static com.codahale.metrics.MetricRegistry.name;
+import com.codahale.metrics.graphite.Graphite;
+import com.codahale.metrics.graphite.GraphiteReporter;
+
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.apache.bookkeeper.stats.StatsProvider;
+import org.apache.commons.configuration.Configuration;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+
+import java.io.File;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CodahaleMetricsProvider implements StatsProvider {
+
+ static final Logger LOG = LoggerFactory.getLogger(CodahaleMetricsProvider.class);
+
+ MetricRegistry metrics = null;
+ List<ScheduledReporter> reporters = new ArrayList<ScheduledReporter>();
+
+ synchronized void initIfNecessary() {
+ if (metrics == null) {
+ metrics = new MetricRegistry();
+ metrics.registerAll(new MemoryUsageGaugeSet());
+ metrics.registerAll(new GarbageCollectorMetricSet());
+ }
+ }
+
+ @Override
+ public void start(Configuration conf) {
+ initIfNecessary();
+
+ int metricsOutputFrequency = conf.getInt("codahaleStatsOutputFrequencySeconds", 60);
+ String prefix = conf.getString("codahaleStatsPrefix", "");
+ String graphiteHost = conf.getString("codahaleStatsGraphiteEndpoint");
+ String csvDir = conf.getString("codahaleStatsCSVEndpoint");
+ String slf4jCat = conf.getString("codahaleStatsSlf4jEndpoint");
+
+ if (!Strings.isNullOrEmpty(graphiteHost)) {
+ LOG.info("Configuring stats with graphite");
+ HostAndPort addr = HostAndPort.fromString(graphiteHost);
+ final Graphite graphite = new Graphite(
+ new InetSocketAddress(addr.getHostText(), addr.getPort()));
+ reporters.add(GraphiteReporter.forRegistry(metrics)
+ .prefixedWith(prefix)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .filter(MetricFilter.ALL)
+ .build(graphite));
+ }
+ if (!Strings.isNullOrEmpty(csvDir)) {
+ // NOTE: 1/ metrics output files are exclusive to a given process
+ // 2/ the output directory must exist
+ // 3/ if output files already exist they are not overwritten and there is no metrics output
+ File outdir;
+ if (Strings.isNullOrEmpty(prefix)) {
+ outdir = new File(csvDir, prefix);
+ } else {
+ outdir = new File(csvDir);
+ }
+ LOG.info("Configuring stats with csv output to directory [{}]", outdir.getAbsolutePath());
+ reporters.add(CsvReporter.forRegistry(metrics)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build(outdir));
+ }
+ if (!Strings.isNullOrEmpty(slf4jCat)) {
+ LOG.info("Configuring stats with slf4j");
+ reporters.add(Slf4jReporter.forRegistry(metrics)
+ .outputTo(LoggerFactory.getLogger(slf4jCat))
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build());
+ }
+ for (ScheduledReporter r : reporters) {
+ r.start(metricsOutputFrequency, TimeUnit.SECONDS);
+ }
+ }
+
+ @Override
+ public void stop() {
+ for (ScheduledReporter r : reporters) {
+ r.report();
+ r.stop();
+ }
+ }
+
+ @Override
+ public StatsLogger getStatsLogger(String name) {
+ initIfNecessary();
+ return new CodahaleStatsLogger(metrics, name);
+ }
+}
Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java?rev=1540366&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java Sat Nov 9 18:49:55 2013
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import org.apache.bookkeeper.stats.OpStatsData;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.Snapshot;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+class CodahaleOpStatsLogger implements OpStatsLogger {
+ final Timer success;
+ final Timer fail;
+
+ CodahaleOpStatsLogger(Timer success, Timer fail) {
+ this.success = success;
+ this.fail = fail;
+ }
+
+ // OpStatsLogger functions
+ public void registerFailedEvent(long eventLatencyMillis) {
+ fail.update(eventLatencyMillis, TimeUnit.MILLISECONDS);
+ }
+
+ public void registerSuccessfulEvent(long eventLatencyMillis) {
+ success.update(eventLatencyMillis, TimeUnit.MILLISECONDS);
+ }
+
+ public synchronized void clear() {
+ // can't clear a timer
+ }
+
+ /**
+ * This function should go away soon (hopefully).
+ */
+ public synchronized OpStatsData toOpStatsData() {
+ long numFailed = fail.getCount();
+ long numSuccess = success.getCount();
+ Snapshot s = success.getSnapshot();
+ double avgLatencyMillis = s.getMean();
+
+ double[] default_percentiles = {10, 50, 90, 99, 99.9, 99.99};
+ long[] latenciesMillis = new long[default_percentiles.length];
+ Arrays.fill(latenciesMillis, Long.MAX_VALUE);
+ for (int i = 0; i < default_percentiles.length; i++) {
+ latenciesMillis[i] = (long)s.getValue(default_percentiles[i]);
+ }
+ return new OpStatsData(numSuccess, numFailed, avgLatencyMillis, latenciesMillis);
+ }
+}
Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java?rev=1540366&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java Sat Nov 9 18:49:55 2013
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.MetricRegistry;
+import static com.codahale.metrics.MetricRegistry.name;
+
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.Gauge;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+
+public class CodahaleStatsLogger implements StatsLogger {
+ protected final String basename;
+ final MetricRegistry metrics;
+
+ CodahaleStatsLogger(MetricRegistry metrics, String basename) {
+ this.metrics = metrics;
+ this.basename = basename;
+ }
+
+ @Override
+ public OpStatsLogger getOpStatsLogger(String statName) {
+ Timer success = metrics.timer(name(basename, statName));
+ Timer failure = metrics.timer(name(basename, statName+"-fail"));
+ return new CodahaleOpStatsLogger(success, failure);
+ }
+
+ @Override
+ public Counter getCounter(String statName) {
+ final com.codahale.metrics.Counter c = metrics.counter(name(basename, statName));
+ return new Counter() {
+ @Override
+ public synchronized void clear() {
+ long cur = c.getCount();
+ c.dec(cur);
+ }
+
+ @Override
+ public Long get() {
+ return c.getCount();
+ }
+
+ @Override
+ public void inc() {
+ c.inc();
+ }
+
+ @Override
+ public void dec() {
+ c.dec();
+ }
+
+ @Override
+ public void add(long delta) {
+ c.inc(delta);
+ }
+ };
+ }
+
+ @Override
+ public <T extends Number> void registerGauge(final String statName, final Gauge<T> gauge) {
+ String metricName = name(basename, statName);
+ metrics.remove(metricName);
+
+ metrics.register(metricName, new com.codahale.metrics.Gauge<T>() {
+ @Override
+ public T getValue() {
+ return gauge.getSample();
+ }
+ });
+ }
+
+ @Override
+ public StatsLogger scope(String scope) {
+ String scopeName;
+ if (0 == basename.length()) {
+ scopeName = scope;
+ } else {
+ scopeName = name(basename, scope);
+ }
+ return new CodahaleStatsLogger(metrics, scopeName);
+ }
+}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml?rev=1540366&r1=1540365&r2=1540366&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml Sat Nov 9 18:49:55 2013
@@ -33,6 +33,7 @@
<modules>
<module>twitter-science-provider</module>
<module>twitter-ostrich-provider</module>
+ <module>codahale-metrics-provider</module>
</modules>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>