You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sa...@apache.org on 2015/05/21 14:47:09 UTC
[3/6] cassandra git commit: Improve row count estimate
Improve row count estimate
Patch by Chris Lohfink; reviewed by Sam Tunnicliffe & Aleksey Yeschenko
for CASSANDRA-9107
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ae048613
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ae048613
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ae048613
Branch: refs/heads/trunk
Commit: ae0486132e5aa7e49868aa36945829ce59067815
Parents: 2248780
Author: Chris Lohfink <Ch...@datastax.com>
Authored: Thu Apr 2 16:50:04 2015 -0500
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu May 21 09:58:29 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 1 -
src/java/org/apache/cassandra/db/Memtable.java | 5 +++++
.../apache/cassandra/metrics/ColumnFamilyMetrics.java | 14 ++++++++++++++
src/java/org/apache/cassandra/tools/NodeProbe.java | 1 +
src/java/org/apache/cassandra/tools/NodeTool.java | 5 +----
6 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ae048613/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 92d4d54..ca12522 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.6
+ * Improve estimated row count (CASSANDRA-9107)
* Optimize range tombstone memory footprint (CASSANDRA-8603)
* Use configured gcgs in anticompaction (CASSANDRA-9397)
* Warn on misuse of unlogged batches (CASSANDRA-9282)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ae048613/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 0951c01..06fc89e 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1180,7 +1180,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public void apply(DecoratedKey key, ColumnFamily columnFamily, SecondaryIndexManager.Updater indexer, OpOrder.Group opGroup, ReplayPosition replayPosition)
{
long start = System.nanoTime();
-
Memtable mt = data.getMemtableFor(opGroup, replayPosition);
final long timeDelta = mt.put(key, columnFamily, indexer, opGroup);
maybeUpdateRowCache(key);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ae048613/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java
index 9a8596a..a50a614 100644
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@ -226,6 +226,11 @@ public class Memtable
return builder.toString();
}
+ public int partitionCount()
+ {
+ return rows.size();
+ }
+
public FlushRunnable flushRunnable()
{
return new FlushRunnable(lastReplayPosition.get());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ae048613/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index c82569d..c4a2aa2 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Memtable;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.utils.EstimatedHistogram;
@@ -61,6 +62,8 @@ public class ColumnFamilyMetrics
public final Gauge<Double> compressionRatio;
/** Histogram of estimated row size (in bytes). */
public final Gauge<long[]> estimatedRowSizeHistogram;
+ /** Approximate number of keys in table. */
+ public final Gauge<Long> estimatedRowCount;
/** Histogram of estimated number of columns. */
public final Gauge<long[]> estimatedColumnCountHistogram;
/** Histogram of the number of sstable data files accessed per read */
@@ -283,6 +286,16 @@ public class ColumnFamilyMetrics
});
}
});
+ estimatedRowCount = Metrics.newGauge(factory.createMetricName("EstimatedRowCount"), new Gauge<Long>()
+ {
+ public Long value()
+ {
+ long memtablePartitions = 0;
+ for (Memtable memtable : cfs.getDataTracker().getView().getAllMemtables())
+ memtablePartitions += memtable.partitionCount();
+ return SSTableReader.getApproximateKeyCount(cfs.getSSTables()) + memtablePartitions;
+ }
+ });
estimatedColumnCountHistogram = Metrics.newGauge(factory.createMetricName("EstimatedColumnCountHistogram"), new Gauge<long[]>()
{
public long[] value()
@@ -624,6 +637,7 @@ public class ColumnFamilyMetrics
writeLatency.release();
rangeLatency.release();
Metrics.defaultRegistry().removeMetric(factory.createMetricName("EstimatedRowSizeHistogram"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("EstimatedRowCount"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("EstimatedColumnCountHistogram"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("CoordinatorReadLatency"));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ae048613/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index 6e7179a..da1fca0 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -1112,6 +1112,7 @@ public class NodeProbe implements AutoCloseable
case "CompressionRatio":
case "EstimatedColumnCountHistogram":
case "EstimatedRowSizeHistogram":
+ case "EstimatedRowCount":
case "KeyCacheHitRate":
case "LiveSSTableCount":
case "MaxRowSize":
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ae048613/src/java/org/apache/cassandra/tools/NodeTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index 5156479..86b5f52 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -867,10 +867,7 @@ public class NodeTool
if (offHeapSize != null)
System.out.println("\t\tOff heap memory used (total): " + format(offHeapSize, humanReadable));
System.out.println("\t\tSSTable Compression Ratio: " + probe.getColumnFamilyMetric(keyspaceName, cfName, "CompressionRatio"));
- long numberOfKeys = 0;
- for (long keys : (long[]) probe.getColumnFamilyMetric(keyspaceName, cfName, "EstimatedColumnCountHistogram"))
- numberOfKeys += keys;
- System.out.println("\t\tNumber of keys (estimate): " + numberOfKeys);
+ System.out.println("\t\tNumber of keys (estimate): " + probe.getColumnFamilyMetric(keyspaceName, cfName, "EstimatedRowCount"));
System.out.println("\t\tMemtable cell count: " + probe.getColumnFamilyMetric(keyspaceName, cfName, "MemtableColumnsCount"));
System.out.println("\t\tMemtable data size: " + format((Long) probe.getColumnFamilyMetric(keyspaceName, cfName, "MemtableLiveDataSize"), humanReadable));
if (memtableOffHeapSize != null)