You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jj...@apache.org on 2017/03/13 04:37:30 UTC
cassandra git commit: Add histogram for delay to deliver hints
Repository: cassandra
Updated Branches:
refs/heads/trunk 67e9a5ffd -> 0c5faef66
Add histogram for delay to deliver hints
Patch by Jeff Jirsa; Reviewed by Stefan Podkowinski for CASSANDRA-13234
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0c5faef6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0c5faef6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0c5faef6
Branch: refs/heads/trunk
Commit: 0c5faef664aa403998f59cc77c9b39861890cfa1
Parents: 67e9a5f
Author: Jeff Jirsa <je...@jeffjirsa.net>
Authored: Fri Feb 17 14:20:49 2017 -0800
Committer: Jeff Jirsa <je...@jeffjirsa.net>
Committed: Sun Mar 12 21:37:11 2017 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
doc/source/operating/metrics.rst | 25 ++++++++++++++++
.../cassandra/hints/EncodedHintMessage.java | 5 ++++
src/java/org/apache/cassandra/hints/Hint.java | 6 ++++
.../apache/cassandra/hints/HintsDispatcher.java | 3 ++
.../cassandra/metrics/HintsServiceMetrics.java | 31 ++++++++++++++++++++
6 files changed, 71 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3acc2b4..7247f36 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -38,6 +38,7 @@
* Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
* cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
* Add support for arithmetic operators (CASSANDRA-11935)
+ * Add histogram for delay to deliver hints (CASSANDRA-13234)
3.11.0
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/doc/source/operating/metrics.rst
----------------------------------------------------------------------
diff --git a/doc/source/operating/metrics.rst b/doc/source/operating/metrics.rst
index 373d4d2..af2e36e 100644
--- a/doc/source/operating/metrics.rst
+++ b/doc/source/operating/metrics.rst
@@ -524,6 +524,31 @@ Hints_created-<PeerIP> Counter Number of hints on disk for this pee
Hints_not_stored-<PeerIP> Counter Number of hints not stored for this peer, due to being down past the configured hint window.
=========================== ============== ===========
+HintsService Metrics
+^^^^^^^^^^^^^^^^^^^^^
+
+Metrics specific to the Hints delivery service. There are also some metrics related to hints tracked in ``Storage Metrics``
+
+These metrics include the peer endpoint **in the metric name**
+
+Reported name format:
+
+**Metric Name**
+ ``org.apache.cassandra.metrics.HintsService.<MetricName>``
+
+**JMX MBean**
+ ``org.apache.cassandra.metrics:type=HintsService name=<MetricName>``
+
+=========================== ============== ===========
+Name Type Description
+=========================== ============== ===========
+HintsSucceeded Meter A meter of the hints successfully delivered
+HintsFailed Meter A meter of the hints that failed deliver
+HintsTimedOut Meter A meter of the hints that timed out
+Hints_delays Histogram Histogram of hint delivery delays (in milliseconds)
+Hints_delays-<PeerIP> Histogram Histogram of hint delivery delays (in milliseconds) per peer
+=========================== ============== ===========
+
SSTable Index Metrics
^^^^^^^^^^^^^^^^^^^^^
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
index 4fe05ac..50d1302 100644
--- a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
+++ b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
@@ -58,6 +58,11 @@ final class EncodedHintMessage
return new MessageOut<>(MessagingService.Verb.HINT, this, serializer);
}
+ public long getHintCreationTime()
+ {
+ return Hint.serializer.getHintCreationTime(hint, version);
+ }
+
private static class Serializer implements IVersionedSerializer<EncodedHintMessage>
{
public long serializedSize(EncodedHintMessage message, int version)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/hints/Hint.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/Hint.java b/src/java/org/apache/cassandra/hints/Hint.java
index 4e8f139..1582a3c 100644
--- a/src/java/org/apache/cassandra/hints/Hint.java
+++ b/src/java/org/apache/cassandra/hints/Hint.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.hints;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@@ -146,5 +147,10 @@ public final class Hint
int gcgs = (int) in.readUnsignedVInt();
return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs);
}
+
+ public long getHintCreationTime(ByteBuffer hintBuffer, int version)
+ {
+ return hintBuffer.getLong(0);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/hints/HintsDispatcher.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java
index 3ac77a3..6fc55df 100644
--- a/src/java/org/apache/cassandra/hints/HintsDispatcher.java
+++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java
@@ -27,6 +27,7 @@ import java.util.function.Function;
import com.google.common.util.concurrent.RateLimiter;
+import org.apache.cassandra.db.monitoring.ApproximateTime;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.metrics.HintsServiceMetrics;
import org.apache.cassandra.net.IAsyncCallbackWithFailure;
@@ -175,6 +176,7 @@ final class HintsDispatcher implements AutoCloseable
{
Callback callback = new Callback();
HintMessage message = new HintMessage(hostId, hint);
+ HintsServiceMetrics.updateDelayMetrics(address , ApproximateTime.currentTimeMillis() - hint.creationTime);
MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback);
return callback;
}
@@ -187,6 +189,7 @@ final class HintsDispatcher implements AutoCloseable
{
Callback callback = new Callback();
EncodedHintMessage message = new EncodedHintMessage(hostId, hint, messagingVersion);
+ HintsServiceMetrics.updateDelayMetrics(address, ApproximateTime.currentTimeMillis() - message.getHintCreationTime());
MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback);
return callback;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
index ad85281..d6a75f7 100644
--- a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
@@ -17,7 +17,16 @@
*/
package org.apache.cassandra.metrics;
+import java.net.InetAddress;
+
+import com.google.common.util.concurrent.MoreExecutors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
@@ -26,9 +35,31 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
*/
public final class HintsServiceMetrics
{
+ private static final Logger logger = LoggerFactory.getLogger(HintsServiceMetrics.class);
+
private static final MetricNameFactory factory = new DefaultNameFactory("HintsService");
public static final Meter hintsSucceeded = Metrics.meter(factory.createMetricName("HintsSucceeded"));
public static final Meter hintsFailed = Metrics.meter(factory.createMetricName("HintsFailed"));
public static final Meter hintsTimedOut = Metrics.meter(factory.createMetricName("HintsTimedOut"));
+
+ /** Histogram of all hint delivery delays */
+ private static final Histogram globalDelayHistogram = Metrics.histogram(factory.createMetricName("Hint_delays"), false);
+
+ /** Histograms per-endpoint of hint delivery delays, This is not a cache. */
+ private static final LoadingCache<InetAddress, Histogram> delayByEndpoint = Caffeine.newBuilder()
+ .executor(MoreExecutors.directExecutor())
+ .build(address -> Metrics.histogram(factory.createMetricName("Hint_delays-"+address.getHostAddress().replace(':', '.')), false));
+
+ public static void updateDelayMetrics(InetAddress endpoint, long delay)
+ {
+ if (delay <= 0)
+ {
+ logger.warn("Invalid negative latency in hint delivery delay: {}", delay);
+ return;
+ }
+
+ globalDelayHistogram.update(delay);
+ delayByEndpoint.get(endpoint).update(delay);
+ }
}