You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jm...@apache.org on 2014/11/06 17:20:48 UTC
git commit: Fix overflow on histogram computation
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 85ea37356 -> 9685622b3
Fix overflow on histogram computation
Patch by cyeksigian; reviewed by jmckenzie for CASSANDRA-8028
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9685622b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9685622b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9685622b
Branch: refs/heads/cassandra-2.1
Commit: 9685622b32d19ffc9315e764a224f41cbe6b6d4a
Parents: 85ea373
Author: Carl Yeksigian <ca...@apache.org>
Authored: Thu Nov 6 10:18:09 2014 -0600
Committer: Joshua McKenzie <jm...@apache.org>
Committed: Thu Nov 6 10:18:09 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/metrics/ColumnFamilyMetrics.java | 67 ++++++++++++++++----
.../org/apache/cassandra/tools/NodeTool.java | 32 ++++++++--
3 files changed, 80 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9685622b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5348f2f..89db48e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.3
+ * Fix overflow on histogram computation (CASSANDRA-8028)
* Have paxos reuse the timestamp generation of normal queries (CASSANDRA-7801)
2.1.2
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9685622b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index d9d3ed9..2bdaf27 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.metrics;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
@@ -146,6 +147,46 @@ public class ColumnFamilyMetrics
* Stores all metric names created that can be used when unregistering
*/
public final static Set<String> all = Sets.newHashSet();
+
+ private interface GetHistogram
+ {
+ public EstimatedHistogram getHistogram(SSTableReader reader);
+ }
+
+ private static long[] combineHistograms(Iterable<SSTableReader> sstables, GetHistogram getHistogram)
+ {
+ Iterator<SSTableReader> iterator = sstables.iterator();
+ if (!iterator.hasNext())
+ {
+ return new long[0];
+ }
+ long[] firstBucket = getHistogram.getHistogram(iterator.next()).getBuckets(false);
+ long[] values = new long[firstBucket.length];
+ System.arraycopy(firstBucket, 0, values, 0, values.length);
+
+ while (iterator.hasNext())
+ {
+ long[] nextBucket = getHistogram.getHistogram(iterator.next()).getBuckets(false);
+ if (nextBucket.length > values.length)
+ {
+ long[] newValues = new long[nextBucket.length];
+ System.arraycopy(firstBucket, 0, newValues, 0, firstBucket.length);
+ for (int i = 0; i < newValues.length; i++)
+ {
+ newValues[i] += nextBucket[i];
+ }
+ values = newValues;
+ }
+ else
+ {
+ for (int i = 0; i < values.length; i++)
+ {
+ values[i] += nextBucket[i];
+ }
+ }
+ }
+ return values;
+ }
/**
* Creates metrics for given {@link ColumnFamilyStore}.
@@ -219,28 +260,26 @@ public class ColumnFamilyMetrics
{
public long[] value()
{
- long[] histogram = new long[90];
- for (SSTableReader sstable : cfs.getSSTables())
+ return combineHistograms(cfs.getSSTables(), new GetHistogram()
{
- long[] rowSize = sstable.getEstimatedRowSize().getBuckets(false);
- for (int i = 0; i < histogram.length; i++)
- histogram[i] += rowSize[i];
- }
- return histogram;
+ public EstimatedHistogram getHistogram(SSTableReader reader)
+ {
+ return reader.getEstimatedRowSize();
+ }
+ });
}
});
estimatedColumnCountHistogram = Metrics.newGauge(factory.createMetricName("EstimatedColumnCountHistogram"), new Gauge<long[]>()
{
public long[] value()
{
- long[] histogram = new long[90];
- for (SSTableReader sstable : cfs.getSSTables())
+ return combineHistograms(cfs.getSSTables(), new GetHistogram()
{
- long[] columnSize = sstable.getEstimatedColumnCount().getBuckets(false);
- for (int i = 0; i < histogram.length; i++)
- histogram[i] += columnSize[i];
- }
- return histogram;
+ public EstimatedHistogram getHistogram(SSTableReader reader)
+ {
+ return reader.getEstimatedColumnCount();
+ }
+ });
}
});
sstablesPerReadHistogram = createColumnFamilyHistogram("SSTablesPerReadHistogram", cfs.keyspace.metric.sstablesPerReadHistogram);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9685622b/src/java/org/apache/cassandra/tools/NodeTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index 39bc4fd..c09751b 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -895,18 +895,38 @@ public class NodeTool
long[] estimatedRowSize = (long[]) probe.getColumnFamilyMetric(keyspace, cfname, "EstimatedRowSizeHistogram");
long[] estimatedColumnCount = (long[]) probe.getColumnFamilyMetric(keyspace, cfname, "EstimatedColumnCountHistogram");
- long[] bucketOffsets = new EstimatedHistogram().getBucketOffsets();
- EstimatedHistogram rowSizeHist = new EstimatedHistogram(bucketOffsets, estimatedRowSize);
- EstimatedHistogram columnCountHist = new EstimatedHistogram(bucketOffsets, estimatedColumnCount);
+ long[] rowSizeBucketOffsets = new EstimatedHistogram(estimatedRowSize.length).getBucketOffsets();
+ long[] columnCountBucketOffsets = new EstimatedHistogram(estimatedColumnCount.length).getBucketOffsets();
+ EstimatedHistogram rowSizeHist = new EstimatedHistogram(rowSizeBucketOffsets, estimatedRowSize);
+ EstimatedHistogram columnCountHist = new EstimatedHistogram(columnCountBucketOffsets, estimatedColumnCount);
// build arrays to store percentile values
double[] estimatedRowSizePercentiles = new double[7];
double[] estimatedColumnCountPercentiles = new double[7];
double[] offsetPercentiles = new double[]{0.5, 0.75, 0.95, 0.98, 0.99};
- for (int i = 0; i < offsetPercentiles.length; i++)
+
+ if (rowSizeHist.isOverflowed())
+ {
+ System.err.println(String.format("Row sizes are larger than %s, unable to calculate percentiles", rowSizeBucketOffsets[rowSizeBucketOffsets.length - 1]));
+ for (int i = 0; i < offsetPercentiles.length; i++)
+ estimatedRowSizePercentiles[i] = Double.NaN;
+ }
+ else
+ {
+ for (int i = 0; i < offsetPercentiles.length; i++)
+ estimatedRowSizePercentiles[i] = rowSizeHist.percentile(offsetPercentiles[i]);
+ }
+
+ if (columnCountHist.isOverflowed())
+ {
+ System.err.println(String.format("Column counts are larger than %s, unable to calculate percentiles", columnCountBucketOffsets[columnCountBucketOffsets.length - 1]));
+ for (int i = 0; i < estimatedColumnCountPercentiles.length; i++)
+ estimatedColumnCountPercentiles[i] = Double.NaN;
+ }
+ else
{
- estimatedRowSizePercentiles[i] = rowSizeHist.percentile(offsetPercentiles[i]);
- estimatedColumnCountPercentiles[i] = columnCountHist.percentile(offsetPercentiles[i]);
+ for (int i = 0; i < offsetPercentiles.length; i++)
+ estimatedColumnCountPercentiles[i] = columnCountHist.percentile(offsetPercentiles[i]);
}
// min value