You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2013/09/23 20:33:52 UTC
[1/6] git commit: Add tombstone debug threshold and histogram patch
by Russell Spitzer and Lyuben Todorov;
reviewed by jbellis for CASSANDRA-6042 and CASSANDRA-6057
Updated Branches:
refs/heads/cassandra-1.2 2980796a7 -> 2267c2094
refs/heads/cassandra-2.0 fb5584979 -> 52b4fc393
refs/heads/trunk 74e710a12 -> eb125ad78
Add tombstone debug threshold and histogram
patch by Russell Spitzer and Lyuben Todorov; reviewed by jbellis for CASSANDRA-6042 and CASSANDRA-6057
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2267c209
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2267c209
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2267c209
Branch: refs/heads/cassandra-1.2
Commit: 2267c20942feb486fbbf1bd54d7a0fbbe468ae10
Parents: 2980796
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Sep 23 09:30:22 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Mon Sep 23 12:39:46 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 5 ++++
.../org/apache/cassandra/config/Config.java | 2 ++
.../cassandra/config/DatabaseDescriptor.java | 14 ++++++++++
.../apache/cassandra/db/ColumnFamilyStore.java | 28 +++++++++++++++++---
.../cassandra/db/ColumnFamilyStoreMBean.java | 9 +++++++
.../cassandra/db/filter/SliceQueryFilter.java | 19 +++++++++++--
.../cassandra/metrics/ColumnFamilyMetrics.java | 8 ++++++
.../cassandra/service/StorageService.java | 10 +++++++
.../cassandra/service/StorageServiceMBean.java | 5 ++++
10 files changed, 95 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 124b3e4..9ed00e2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,7 @@
* Allow where clause conditions to be in parenthesis (CASSANDRA-6037)
* Do not open non-ssl storage port if encryption option is all (CASSANDRA-3916)
* Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+ * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057)
1.2.10
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 27ac09b..a3cdec7 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -440,6 +440,11 @@ snapshot_before_compaction: false
# lose data on truncation or drop.
auto_snapshot: true
+# Log a debug message if more than this many tombstones are scanned
+# in a single-partition query. Set the threshold on SliceQueryFilter
+# to debug to enable.
+tombstone_debug_threshold: 10000
+
# Add column indexes to a row after its contents reach this size.
# Increase if your column values are large, or if you have a very large
# number of columns. The competing causes are, Cassandra has to
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index a924a4c..292161b 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -176,6 +176,8 @@ public class Config
private static boolean loadYaml = true;
private static boolean outboundBindAny = false;
+ public volatile int tombstone_debug_threshold = 10000;
+
public static boolean getOutboundBindAny()
{
return outboundBindAny;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 22de2d6..2c999ef 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -930,6 +930,20 @@ public class DatabaseDescriptor
}
/**
+ * How many tombstones need to be scanned before we log a
+ * debug message
+ */
+ public static int getTombstoneDebugThreshold()
+ {
+ return conf.tombstone_debug_threshold;
+ }
+
+ public static void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+ {
+ conf.tombstone_debug_threshold = tombstoneDebugThreshold;
+ }
+
+ /**
* size of commitlog segments to allocate
*/
public static int getCommitLogSegmentSize()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index c646461..a41c157 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -49,10 +49,7 @@ import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.filter.ExtendedFilter;
-import org.apache.cassandra.db.filter.IDiskAtomFilter;
-import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -1221,6 +1218,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
result = cf.isSuper() ? removeDeleted(cf, gcBefore) : removeDeletedCF(cf, gcBefore);
}
+
+ if (filter.filter instanceof SliceQueryFilter)
+ {
+ // Log the number of tombstones scanned on single key queries
+ metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored());
+ metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive());
+ }
}
finally
{
@@ -1907,6 +1911,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return getMinimumCompactionThreshold() <= 0 || getMaximumCompactionThreshold() <= 0;
}
+ public long getTombstonesPerLastRead()
+ {
+ return metric.tombstoneScannedHistogram.count();
+ }
+
+ public float getPercentageTombstonesPerLastRead()
+ {
+ long total = metric.tombstoneScannedHistogram.count() + metric.liveScannedHistogram.count();
+ return (metric.tombstoneScannedHistogram.count() / total);
+ }
+
+ public long getLiveCellsPerLastRead()
+ {
+ return metric.liveScannedHistogram.count();
+ }
+
// End JMX get/set.
public long estimateKeys()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index 554c204..f937032 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -286,6 +286,15 @@ public interface ColumnFamilyStoreMBean
*/
public void disableAutoCompaction();
+ /** Number of tombstoned cells retreived during the last slicequery */
+ public long getTombstonesPerLastRead();
+
+ /** Percentage of tombstoned cells retreived during the last slicequery */
+ public float getPercentageTombstonesPerLastRead();
+
+ /** Number of live cells retreived during the last slicequery */
+ public long getLiveCellsPerLastRead();
+
public long estimateKeys();
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 39d1ca1..7e1ec6b 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -17,7 +17,9 @@
*/
package org.apache.cassandra.db.filter;
-import java.io.*;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
@@ -26,9 +28,10 @@ import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.columniterator.ISSTableColumnIterator;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.columniterator.SSTableSliceIterator;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CompositeType;
@@ -172,6 +175,8 @@ public class SliceQueryFilter implements IDiskAtomFilter
}
Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
+ if (columnCounter.ignored() > DatabaseDescriptor.getTombstoneDebugThreshold())
+ logger.debug("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
}
public int getLiveCount(ColumnFamily cf)
@@ -249,6 +254,16 @@ public class SliceQueryFilter implements IDiskAtomFilter
return columnCounter == null ? 0 : columnCounter.live();
}
+ public int lastIgnored()
+ {
+ return columnCounter == null ? 0 : columnCounter.ignored();
+ }
+
+ public int lastLive()
+ {
+ return columnCounter == null ? 0 : columnCounter.live();
+ }
+
@Override
public String toString()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index d111667..9bd90a9 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -79,6 +79,10 @@ public class ColumnFamilyMetrics
public final Gauge<Long> bloomFilterDiskSpaceUsed;
/** Key cache hit rate for this CF */
public final Gauge<Double> keyCacheHitRate;
+ /** Tombstones scanned in queries on this CF */
+ public final Histogram tombstoneScannedHistogram;
+ /** Live cells scanned in queries on this CF */
+ public final Histogram liveScannedHistogram;
private final MetricNameFactory factory;
@@ -295,6 +299,8 @@ public class ColumnFamilyMetrics
return Math.max(requests, 1); // to avoid NaN.
}
});
+ tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram"));
+ liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"));
}
public void updateSSTableIterated(int count)
@@ -331,6 +337,8 @@ public class ColumnFamilyMetrics
Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram"));
}
class ColumnFamilyMetricNameFactory implements MetricNameFactory
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 7967dee..50719fd 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -3934,4 +3934,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
{
return DatabaseDescriptor.getPartitionerName();
}
+
+ public int getTombstoneDebugThreshold()
+ {
+ return DatabaseDescriptor.getTombstoneDebugThreshold();
+ }
+
+ public void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+ {
+ DatabaseDescriptor.setTombstoneDebugThreshold(tombstoneDebugThreshold);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageServiceMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index 4fbed9c..63add12 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -476,4 +476,9 @@ public interface StorageServiceMBean extends NotificationEmitter
public String getClusterName();
/** Returns the cluster partitioner */
public String getPartitionerName();
+
+ /** Returns the threshold for returning debugging queries with many tombstones */
+ public int getTombstoneDebugThreshold();
+ /** Sets the threshold for returning debugging queries with many tombstones */
+ public void setTombstoneDebugThreshold(int tombstoneDebugThreshold);
}
[3/6] git commit: Add tombstone debug threshold and histogram patch
by Russell Spitzer and Lyuben Todorov;
reviewed by jbellis for CASSANDRA-6042 and CASSANDRA-6057
Posted by jb...@apache.org.
Add tombstone debug threshold and histogram
patch by Russell Spitzer and Lyuben Todorov; reviewed by jbellis for CASSANDRA-6042 and CASSANDRA-6057
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2267c209
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2267c209
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2267c209
Branch: refs/heads/trunk
Commit: 2267c20942feb486fbbf1bd54d7a0fbbe468ae10
Parents: 2980796
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Sep 23 09:30:22 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Mon Sep 23 12:39:46 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 5 ++++
.../org/apache/cassandra/config/Config.java | 2 ++
.../cassandra/config/DatabaseDescriptor.java | 14 ++++++++++
.../apache/cassandra/db/ColumnFamilyStore.java | 28 +++++++++++++++++---
.../cassandra/db/ColumnFamilyStoreMBean.java | 9 +++++++
.../cassandra/db/filter/SliceQueryFilter.java | 19 +++++++++++--
.../cassandra/metrics/ColumnFamilyMetrics.java | 8 ++++++
.../cassandra/service/StorageService.java | 10 +++++++
.../cassandra/service/StorageServiceMBean.java | 5 ++++
10 files changed, 95 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 124b3e4..9ed00e2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,7 @@
* Allow where clause conditions to be in parenthesis (CASSANDRA-6037)
* Do not open non-ssl storage port if encryption option is all (CASSANDRA-3916)
* Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+ * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057)
1.2.10
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 27ac09b..a3cdec7 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -440,6 +440,11 @@ snapshot_before_compaction: false
# lose data on truncation or drop.
auto_snapshot: true
+# Log a debug message if more than this many tombstones are scanned
+# in a single-partition query. Set the threshold on SliceQueryFilter
+# to debug to enable.
+tombstone_debug_threshold: 10000
+
# Add column indexes to a row after its contents reach this size.
# Increase if your column values are large, or if you have a very large
# number of columns. The competing causes are, Cassandra has to
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index a924a4c..292161b 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -176,6 +176,8 @@ public class Config
private static boolean loadYaml = true;
private static boolean outboundBindAny = false;
+ public volatile int tombstone_debug_threshold = 10000;
+
public static boolean getOutboundBindAny()
{
return outboundBindAny;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 22de2d6..2c999ef 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -930,6 +930,20 @@ public class DatabaseDescriptor
}
/**
+ * How many tombstones need to be scanned before we log a
+ * debug message
+ */
+ public static int getTombstoneDebugThreshold()
+ {
+ return conf.tombstone_debug_threshold;
+ }
+
+ public static void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+ {
+ conf.tombstone_debug_threshold = tombstoneDebugThreshold;
+ }
+
+ /**
* size of commitlog segments to allocate
*/
public static int getCommitLogSegmentSize()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index c646461..a41c157 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -49,10 +49,7 @@ import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.filter.ExtendedFilter;
-import org.apache.cassandra.db.filter.IDiskAtomFilter;
-import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -1221,6 +1218,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
result = cf.isSuper() ? removeDeleted(cf, gcBefore) : removeDeletedCF(cf, gcBefore);
}
+
+ if (filter.filter instanceof SliceQueryFilter)
+ {
+ // Log the number of tombstones scanned on single key queries
+ metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored());
+ metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive());
+ }
}
finally
{
@@ -1907,6 +1911,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return getMinimumCompactionThreshold() <= 0 || getMaximumCompactionThreshold() <= 0;
}
+ public long getTombstonesPerLastRead()
+ {
+ return metric.tombstoneScannedHistogram.count();
+ }
+
+ public float getPercentageTombstonesPerLastRead()
+ {
+ long total = metric.tombstoneScannedHistogram.count() + metric.liveScannedHistogram.count();
+ return (metric.tombstoneScannedHistogram.count() / total);
+ }
+
+ public long getLiveCellsPerLastRead()
+ {
+ return metric.liveScannedHistogram.count();
+ }
+
// End JMX get/set.
public long estimateKeys()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index 554c204..f937032 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -286,6 +286,15 @@ public interface ColumnFamilyStoreMBean
*/
public void disableAutoCompaction();
+ /** Number of tombstoned cells retreived during the last slicequery */
+ public long getTombstonesPerLastRead();
+
+ /** Percentage of tombstoned cells retreived during the last slicequery */
+ public float getPercentageTombstonesPerLastRead();
+
+ /** Number of live cells retreived during the last slicequery */
+ public long getLiveCellsPerLastRead();
+
public long estimateKeys();
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 39d1ca1..7e1ec6b 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -17,7 +17,9 @@
*/
package org.apache.cassandra.db.filter;
-import java.io.*;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
@@ -26,9 +28,10 @@ import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.columniterator.ISSTableColumnIterator;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.columniterator.SSTableSliceIterator;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CompositeType;
@@ -172,6 +175,8 @@ public class SliceQueryFilter implements IDiskAtomFilter
}
Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
+ if (columnCounter.ignored() > DatabaseDescriptor.getTombstoneDebugThreshold())
+ logger.debug("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
}
public int getLiveCount(ColumnFamily cf)
@@ -249,6 +254,16 @@ public class SliceQueryFilter implements IDiskAtomFilter
return columnCounter == null ? 0 : columnCounter.live();
}
+ public int lastIgnored()
+ {
+ return columnCounter == null ? 0 : columnCounter.ignored();
+ }
+
+ public int lastLive()
+ {
+ return columnCounter == null ? 0 : columnCounter.live();
+ }
+
@Override
public String toString()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index d111667..9bd90a9 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -79,6 +79,10 @@ public class ColumnFamilyMetrics
public final Gauge<Long> bloomFilterDiskSpaceUsed;
/** Key cache hit rate for this CF */
public final Gauge<Double> keyCacheHitRate;
+ /** Tombstones scanned in queries on this CF */
+ public final Histogram tombstoneScannedHistogram;
+ /** Live cells scanned in queries on this CF */
+ public final Histogram liveScannedHistogram;
private final MetricNameFactory factory;
@@ -295,6 +299,8 @@ public class ColumnFamilyMetrics
return Math.max(requests, 1); // to avoid NaN.
}
});
+ tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram"));
+ liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"));
}
public void updateSSTableIterated(int count)
@@ -331,6 +337,8 @@ public class ColumnFamilyMetrics
Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram"));
}
class ColumnFamilyMetricNameFactory implements MetricNameFactory
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 7967dee..50719fd 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -3934,4 +3934,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
{
return DatabaseDescriptor.getPartitionerName();
}
+
+ public int getTombstoneDebugThreshold()
+ {
+ return DatabaseDescriptor.getTombstoneDebugThreshold();
+ }
+
+ public void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+ {
+ DatabaseDescriptor.setTombstoneDebugThreshold(tombstoneDebugThreshold);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageServiceMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index 4fbed9c..63add12 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -476,4 +476,9 @@ public interface StorageServiceMBean extends NotificationEmitter
public String getClusterName();
/** Returns the cluster partitioner */
public String getPartitionerName();
+
+ /** Returns the threshold for returning debugging queries with many tombstones */
+ public int getTombstoneDebugThreshold();
+ /** Sets the threshold for returning debugging queries with many tombstones */
+ public void setTombstoneDebugThreshold(int tombstoneDebugThreshold);
}
[2/6] git commit: Add tombstone debug threshold and histogram patch
by Russell Spitzer and Lyuben Todorov;
reviewed by jbellis for CASSANDRA-6042 and CASSANDRA-6057
Posted by jb...@apache.org.
Add tombstone debug threshold and histogram
patch by Russell Spitzer and Lyuben Todorov; reviewed by jbellis for CASSANDRA-6042 and CASSANDRA-6057
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2267c209
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2267c209
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2267c209
Branch: refs/heads/cassandra-2.0
Commit: 2267c20942feb486fbbf1bd54d7a0fbbe468ae10
Parents: 2980796
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Sep 23 09:30:22 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Mon Sep 23 12:39:46 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 5 ++++
.../org/apache/cassandra/config/Config.java | 2 ++
.../cassandra/config/DatabaseDescriptor.java | 14 ++++++++++
.../apache/cassandra/db/ColumnFamilyStore.java | 28 +++++++++++++++++---
.../cassandra/db/ColumnFamilyStoreMBean.java | 9 +++++++
.../cassandra/db/filter/SliceQueryFilter.java | 19 +++++++++++--
.../cassandra/metrics/ColumnFamilyMetrics.java | 8 ++++++
.../cassandra/service/StorageService.java | 10 +++++++
.../cassandra/service/StorageServiceMBean.java | 5 ++++
10 files changed, 95 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 124b3e4..9ed00e2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,7 @@
* Allow where clause conditions to be in parenthesis (CASSANDRA-6037)
* Do not open non-ssl storage port if encryption option is all (CASSANDRA-3916)
* Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+ * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057)
1.2.10
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 27ac09b..a3cdec7 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -440,6 +440,11 @@ snapshot_before_compaction: false
# lose data on truncation or drop.
auto_snapshot: true
+# Log a debug message if more than this many tombstones are scanned
+# in a single-partition query. Set the threshold on SliceQueryFilter
+# to debug to enable.
+tombstone_debug_threshold: 10000
+
# Add column indexes to a row after its contents reach this size.
# Increase if your column values are large, or if you have a very large
# number of columns. The competing causes are, Cassandra has to
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index a924a4c..292161b 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -176,6 +176,8 @@ public class Config
private static boolean loadYaml = true;
private static boolean outboundBindAny = false;
+ public volatile int tombstone_debug_threshold = 10000;
+
public static boolean getOutboundBindAny()
{
return outboundBindAny;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 22de2d6..2c999ef 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -930,6 +930,20 @@ public class DatabaseDescriptor
}
/**
+ * How many tombstones need to be scanned before we log a
+ * debug message
+ */
+ public static int getTombstoneDebugThreshold()
+ {
+ return conf.tombstone_debug_threshold;
+ }
+
+ public static void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+ {
+ conf.tombstone_debug_threshold = tombstoneDebugThreshold;
+ }
+
+ /**
* size of commitlog segments to allocate
*/
public static int getCommitLogSegmentSize()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index c646461..a41c157 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -49,10 +49,7 @@ import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.filter.ExtendedFilter;
-import org.apache.cassandra.db.filter.IDiskAtomFilter;
-import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -1221,6 +1218,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
result = cf.isSuper() ? removeDeleted(cf, gcBefore) : removeDeletedCF(cf, gcBefore);
}
+
+ if (filter.filter instanceof SliceQueryFilter)
+ {
+ // Log the number of tombstones scanned on single key queries
+ metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored());
+ metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive());
+ }
}
finally
{
@@ -1907,6 +1911,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return getMinimumCompactionThreshold() <= 0 || getMaximumCompactionThreshold() <= 0;
}
+ public long getTombstonesPerLastRead()
+ {
+ return metric.tombstoneScannedHistogram.count();
+ }
+
+ public float getPercentageTombstonesPerLastRead()
+ {
+ long total = metric.tombstoneScannedHistogram.count() + metric.liveScannedHistogram.count();
+ return (metric.tombstoneScannedHistogram.count() / total);
+ }
+
+ public long getLiveCellsPerLastRead()
+ {
+ return metric.liveScannedHistogram.count();
+ }
+
// End JMX get/set.
public long estimateKeys()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index 554c204..f937032 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -286,6 +286,15 @@ public interface ColumnFamilyStoreMBean
*/
public void disableAutoCompaction();
+ /** Number of tombstoned cells retreived during the last slicequery */
+ public long getTombstonesPerLastRead();
+
+ /** Percentage of tombstoned cells retreived during the last slicequery */
+ public float getPercentageTombstonesPerLastRead();
+
+ /** Number of live cells retreived during the last slicequery */
+ public long getLiveCellsPerLastRead();
+
public long estimateKeys();
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 39d1ca1..7e1ec6b 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -17,7 +17,9 @@
*/
package org.apache.cassandra.db.filter;
-import java.io.*;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
@@ -26,9 +28,10 @@ import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.columniterator.ISSTableColumnIterator;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.columniterator.SSTableSliceIterator;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CompositeType;
@@ -172,6 +175,8 @@ public class SliceQueryFilter implements IDiskAtomFilter
}
Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
+ if (columnCounter.ignored() > DatabaseDescriptor.getTombstoneDebugThreshold())
+ logger.debug("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
}
public int getLiveCount(ColumnFamily cf)
@@ -249,6 +254,16 @@ public class SliceQueryFilter implements IDiskAtomFilter
return columnCounter == null ? 0 : columnCounter.live();
}
+ public int lastIgnored()
+ {
+ return columnCounter == null ? 0 : columnCounter.ignored();
+ }
+
+ public int lastLive()
+ {
+ return columnCounter == null ? 0 : columnCounter.live();
+ }
+
@Override
public String toString()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index d111667..9bd90a9 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -79,6 +79,10 @@ public class ColumnFamilyMetrics
public final Gauge<Long> bloomFilterDiskSpaceUsed;
/** Key cache hit rate for this CF */
public final Gauge<Double> keyCacheHitRate;
+ /** Tombstones scanned in queries on this CF */
+ public final Histogram tombstoneScannedHistogram;
+ /** Live cells scanned in queries on this CF */
+ public final Histogram liveScannedHistogram;
private final MetricNameFactory factory;
@@ -295,6 +299,8 @@ public class ColumnFamilyMetrics
return Math.max(requests, 1); // to avoid NaN.
}
});
+ tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram"));
+ liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"));
}
public void updateSSTableIterated(int count)
@@ -331,6 +337,8 @@ public class ColumnFamilyMetrics
Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram"));
}
class ColumnFamilyMetricNameFactory implements MetricNameFactory
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 7967dee..50719fd 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -3934,4 +3934,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
{
return DatabaseDescriptor.getPartitionerName();
}
+
+ public int getTombstoneDebugThreshold()
+ {
+ return DatabaseDescriptor.getTombstoneDebugThreshold();
+ }
+
+ public void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+ {
+ DatabaseDescriptor.setTombstoneDebugThreshold(tombstoneDebugThreshold);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageServiceMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index 4fbed9c..63add12 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -476,4 +476,9 @@ public interface StorageServiceMBean extends NotificationEmitter
public String getClusterName();
/** Returns the cluster partitioner */
public String getPartitionerName();
+
+ /** Returns the threshold for returning debugging queries with many tombstones */
+ public int getTombstoneDebugThreshold();
+ /** Sets the threshold for returning debugging queries with many tombstones */
+ public void setTombstoneDebugThreshold(int tombstoneDebugThreshold);
}
[6/6] git commit: Merge branch 'cassandra-2.0' into trunk
Posted by jb...@apache.org.
Merge branch 'cassandra-2.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/eb125ad7
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/eb125ad7
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/eb125ad7
Branch: refs/heads/trunk
Commit: eb125ad78b57da38026c1f7775744de53d1676a4
Parents: 74e710a 52b4fc3
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Sep 23 13:33:46 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Mon Sep 23 13:33:46 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 5 ++++
.../org/apache/cassandra/config/Config.java | 2 ++
.../cassandra/config/DatabaseDescriptor.java | 14 +++++++++
.../apache/cassandra/db/ColumnFamilyStore.java | 31 ++++++++++++++++----
.../cassandra/db/ColumnFamilyStoreMBean.java | 9 ++++++
.../cassandra/db/filter/SliceQueryFilter.java | 20 +++++++++----
.../cassandra/metrics/ColumnFamilyMetrics.java | 8 +++++
.../cassandra/service/StorageService.java | 10 +++++++
.../cassandra/service/StorageServiceMBean.java | 5 ++++
10 files changed, 93 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eb125ad7/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eb125ad7/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eb125ad7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 36fe7f9,ec9570a..6d27625
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -69,9 -66,9 +66,8 @@@ import org.apache.cassandra.io.util.Fil
import org.apache.cassandra.metrics.ColumnFamilyMetrics;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.thrift.IndexExpression;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.*;
- import org.cliffc.high_scale_lib.NonBlockingHashMap;
import static org.apache.cassandra.config.CFMetaData.Caching;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eb125ad7/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eb125ad7/src/java/org/apache/cassandra/service/StorageServiceMBean.java
----------------------------------------------------------------------
[4/6] git commit: merge from 1.2
Posted by jb...@apache.org.
merge from 1.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/52b4fc39
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/52b4fc39
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/52b4fc39
Branch: refs/heads/trunk
Commit: 52b4fc393350e85f44034094f323f2f06a347968
Parents: fb55849 2267c20
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Sep 23 13:33:38 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Mon Sep 23 13:33:38 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 5 ++++
.../org/apache/cassandra/config/Config.java | 2 ++
.../cassandra/config/DatabaseDescriptor.java | 14 +++++++++
.../apache/cassandra/db/ColumnFamilyStore.java | 31 ++++++++++++++++----
.../cassandra/db/ColumnFamilyStoreMBean.java | 9 ++++++
.../cassandra/db/filter/SliceQueryFilter.java | 20 +++++++++----
.../cassandra/metrics/ColumnFamilyMetrics.java | 8 +++++
.../cassandra/service/StorageService.java | 10 +++++++
.../cassandra/service/StorageServiceMBean.java | 5 ++++
10 files changed, 93 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 88daf35,9ed00e2..aef45ba
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -7,43 -3,10 +7,44 @@@ Merged from 1.2
* Allow where clause conditions to be in parenthesis (CASSANDRA-6037)
* Do not open non-ssl storage port if encryption option is all (CASSANDRA-3916)
* Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+ * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057)
-1.2.10
+2.0.1
+ * Fix bug that could allow reading deleted data temporarily (CASSANDRA-6025)
+ * Improve memory use defaults (CASSANDRA-5069)
+ * Make ThriftServer more easlly extensible (CASSANDRA-6058)
+ * Remove Hadoop dependency from ITransportFactory (CASSANDRA-6062)
+ * add file_cache_size_in_mb setting (CASSANDRA-5661)
+ * Improve error message when yaml contains invalid properties (CASSANDRA-5958)
+ * Improve leveled compaction's ability to find non-overlapping L0 compactions
+ to work on concurrently (CASSANDRA-5921)
+ * Notify indexer of columns shadowed by range tombstones (CASSANDRA-5614)
+ * Log Merkle tree stats (CASSANDRA-2698)
+ * Switch from crc32 to adler32 for compressed sstable checksums (CASSANDRA-5862)
+ * Improve offheap memcpy performance (CASSANDRA-5884)
+ * Use a range aware scanner for cleanup (CASSANDRA-2524)
+ * Cleanup doesn't need to inspect sstables that contain only local data
+ (CASSANDRA-5722)
+ * Add ability for CQL3 to list partition keys (CASSANDRA-4536)
+ * Improve native protocol serialization (CASSANDRA-5664)
+ * Upgrade Thrift to 0.9.1 (CASSANDRA-5923)
+ * Require superuser status for adding triggers (CASSANDRA-5963)
+ * Make standalone scrubber handle old and new style leveled manifest
+ (CASSANDRA-6005)
+ * Fix paxos bugs (CASSANDRA-6012, 6013, 6023)
+ * Fix paged ranges with multiple replicas (CASSANDRA-6004)
+ * Fix potential AssertionError during tracing (CASSANDRA-6041)
+ * Fix NPE in sstablesplit (CASSANDRA-6027)
+ * Migrate pre-2.0 key/value/column aliases to system.schema_columns
+ (CASSANDRA-6009)
+ * Paging filter empty rows too agressively (CASSANDRA-6040)
+ * Support variadic parameters for IN clauses (CASSANDRA-4210)
+ * cqlsh: return the result of CAS writes (CASSANDRA-5796)
+ * Fix validation of IN clauses with 2ndary indexes (CASSANDRA-6050)
+ * Support named bind variables in CQL (CASSANDRA-6033)
+Merged from 1.2:
+ * Allow cache-keys-to-save to be set at runtime (CASSANDRA-5980)
* Avoid second-guessing out-of-space state (CASSANDRA-5605)
* Tuning knobs for dealing with large blobs and many CFs (CASSANDRA-5982)
* (Hadoop) Fix CQLRW for thrift tables (CASSANDRA-6002)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/conf/cassandra.yaml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/Config.java
index 392e8c3,292161b..131c9ef
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@@ -177,8 -173,11 +177,10 @@@ public class Confi
public String memtable_allocator = "SlabAllocator";
- private static boolean loadYaml = true;
private static boolean outboundBindAny = false;
+ public volatile int tombstone_debug_threshold = 10000;
+
public static boolean getOutboundBindAny()
{
return outboundBindAny;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 1ff4832,a41c157..ec9570a
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -28,11 -28,9 +28,12 @@@ import java.util.concurrent.atomic.Atom
import java.util.regex.Pattern;
import javax.management.*;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
import com.google.common.collect.*;
import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.Uninterruptibles;
+ import org.cliffc.high_scale_lib.NonBlockingHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -48,12 -45,11 +49,8 @@@ import org.apache.cassandra.config.Sche
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
-import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.*;
- import org.apache.cassandra.db.filter.ColumnSlice;
- import org.apache.cassandra.db.filter.ExtendedFilter;
- import org.apache.cassandra.db.filter.IDiskAtomFilter;
- import org.apache.cassandra.db.filter.QueryFilter;
- import org.apache.cassandra.db.filter.SliceQueryFilter;
+ import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.db.marshal.AbstractType;
@@@ -1296,10 -1213,18 +1292,17 @@@ public class ColumnFamilyStore implemen
if (cf == null)
return null;
- // TODO this is necessary because when we collate supercolumns together, we don't check
- // their subcolumns for relevance, so we need to do a second prune post facto here.
- result = cf.isSuper() ? removeDeleted(cf, gcBefore) : removeDeletedCF(cf, gcBefore);
-
+ result = removeDeletedCF(cf, gcBefore);
}
+ removeDroppedColumns(result);
++
+ if (filter.filter instanceof SliceQueryFilter)
+ {
+ // Log the number of tombstones scanned on single key queries
+ metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored());
+ metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive());
+ }
}
finally
{
@@@ -2164,15 -1901,32 +2167,31 @@@
private void validateCompactionThresholds(int minThreshold, int maxThreshold)
{
- if (minThreshold > maxThreshold && maxThreshold != 0)
+ if (minThreshold > maxThreshold)
throw new RuntimeException(String.format("The min_compaction_threshold cannot be larger than the max_compaction_threshold. " +
"Min is '%d', Max is '%d'.", minThreshold, maxThreshold));
- }
- public boolean isCompactionDisabled()
- {
- return getMinimumCompactionThreshold() <= 0 || getMaximumCompactionThreshold() <= 0;
+ if (maxThreshold == 0 || minThreshold == 0)
+ throw new RuntimeException("Disabling compaction by setting min_compaction_threshold or max_compaction_threshold to 0 " +
+ "is deprecated, set the compaction strategy option 'enabled' to 'false' instead or use the nodetool command 'disableautocompaction'.");
}
+ public long getTombstonesPerLastRead()
+ {
+ return metric.tombstoneScannedHistogram.count();
+ }
+
+ public float getPercentageTombstonesPerLastRead()
+ {
+ long total = metric.tombstoneScannedHistogram.count() + metric.liveScannedHistogram.count();
+ return (metric.tombstoneScannedHistogram.count() / total);
+ }
+
+ public long getLiveCellsPerLastRead()
+ {
+ return metric.liveScannedHistogram.count();
+ }
+
// End JMX get/set.
public long estimateKeys()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index e810dc6,f937032..6105714
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@@ -281,8 -281,20 +281,17 @@@ public interface ColumnFamilyStoreMBea
*/
public void setCrcCheckChance(double crcCheckChance);
- /**
- * Disable automatic compaction.
- */
- public void disableAutoCompaction();
+ public boolean isAutoCompactionDisabled();
+ /** Number of tombstoned cells retreived during the last slicequery */
+ public long getTombstonesPerLastRead();
+
+ /** Percentage of tombstoned cells retreived during the last slicequery */
+ public float getPercentageTombstonesPerLastRead();
+
+ /** Number of live cells retreived during the last slicequery */
+ public long getLiveCellsPerLastRead();
+
public long estimateKeys();
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 11d7438,7e1ec6b..443ff8e
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@@ -21,17 -21,16 +21,13 @@@ import java.io.DataInput
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
- import java.util.Arrays;
- import java.util.ArrayList;
- import java.util.Collection;
- import java.util.Comparator;
- import java.util.Iterator;
- import java.util.List;
+ import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.ISSTableColumnIterator;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.columniterator.SSTableSliceIterator;
import org.apache.cassandra.db.marshal.AbstractType;
@@@ -202,48 -175,62 +198,50 @@@ public class SliceQueryFilter implement
}
Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
+ if (columnCounter.ignored() > DatabaseDescriptor.getTombstoneDebugThreshold())
+ logger.debug("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
}
- public int getLiveCount(ColumnFamily cf)
+ public int getLiveCount(ColumnFamily cf, long now)
{
- ColumnCounter counter = getColumnCounter(cf);
- for (IColumn column : cf)
- counter.count(column, cf);
- return counter.live();
+ return columnCounter(cf.getComparator(), now).countAll(cf).live();
}
- private ColumnCounter getColumnCounter(IColumnContainer container)
+ public ColumnCounter columnCounter(AbstractType<?> comparator, long now)
{
- AbstractType<?> comparator = container.getComparator();
if (compositesToGroup < 0)
- return new ColumnCounter();
+ return new ColumnCounter(now);
else if (compositesToGroup == 0)
- return new ColumnCounter.GroupByPrefix(null, 0);
+ return new ColumnCounter.GroupByPrefix(now, null, 0);
else
- return new ColumnCounter.GroupByPrefix((CompositeType)comparator, compositesToGroup);
+ return new ColumnCounter.GroupByPrefix(now, (CompositeType)comparator, compositesToGroup);
}
- public void trim(ColumnFamily cf, int trimTo)
+ public void trim(ColumnFamily cf, int trimTo, long now)
{
- ColumnCounter counter = getColumnCounter(cf);
+ ColumnCounter counter = columnCounter(cf.getComparator(), now);
- Collection<ByteBuffer> toRemove = null;
- boolean trimRemaining = false;
+ Collection<Column> columns = reversed
+ ? cf.getReverseSortedColumns()
+ : cf.getSortedColumns();
- Collection<IColumn> columns = reversed
- ? cf.getReverseSortedColumns()
- : cf.getSortedColumns();
+ DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(reversed);
- for (IColumn column : columns)
+ for (Iterator<Column> iter = columns.iterator(); iter.hasNext(); )
{
- if (trimRemaining)
- {
- toRemove.add(column.name());
- continue;
- }
+ Column column = iter.next();
+ counter.count(column, tester);
- counter.count(column, cf);
if (counter.live() > trimTo)
{
- toRemove = new HashSet<ByteBuffer>();
- toRemove.add(column.name());
- trimRemaining = true;
+ iter.remove();
+ while (iter.hasNext())
+ {
+ iter.next();
+ iter.remove();
+ }
}
}
-
- if (toRemove != null)
- {
- for (ByteBuffer columnName : toRemove)
- cf.remove(columnName);
- }
}
public ByteBuffer start()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index 0f9f87d,9bd90a9..b63bbfb
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@@ -334,7 -337,8 +340,9 @@@ public class ColumnFamilyMetric
Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("SpeculativeRetry"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram"));
}
class ColumnFamilyMetricNameFactory implements MetricNameFactory
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/service/StorageServiceMBean.java
----------------------------------------------------------------------
[5/6] git commit: merge from 1.2
Posted by jb...@apache.org.
merge from 1.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/52b4fc39
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/52b4fc39
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/52b4fc39
Branch: refs/heads/cassandra-2.0
Commit: 52b4fc393350e85f44034094f323f2f06a347968
Parents: fb55849 2267c20
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Sep 23 13:33:38 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Mon Sep 23 13:33:38 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 5 ++++
.../org/apache/cassandra/config/Config.java | 2 ++
.../cassandra/config/DatabaseDescriptor.java | 14 +++++++++
.../apache/cassandra/db/ColumnFamilyStore.java | 31 ++++++++++++++++----
.../cassandra/db/ColumnFamilyStoreMBean.java | 9 ++++++
.../cassandra/db/filter/SliceQueryFilter.java | 20 +++++++++----
.../cassandra/metrics/ColumnFamilyMetrics.java | 8 +++++
.../cassandra/service/StorageService.java | 10 +++++++
.../cassandra/service/StorageServiceMBean.java | 5 ++++
10 files changed, 93 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 88daf35,9ed00e2..aef45ba
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -7,43 -3,10 +7,44 @@@ Merged from 1.2
* Allow where clause conditions to be in parenthesis (CASSANDRA-6037)
* Do not open non-ssl storage port if encryption option is all (CASSANDRA-3916)
* Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+ * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057)
-1.2.10
+2.0.1
+ * Fix bug that could allow reading deleted data temporarily (CASSANDRA-6025)
+ * Improve memory use defaults (CASSANDRA-5069)
+ * Make ThriftServer more easlly extensible (CASSANDRA-6058)
+ * Remove Hadoop dependency from ITransportFactory (CASSANDRA-6062)
+ * add file_cache_size_in_mb setting (CASSANDRA-5661)
+ * Improve error message when yaml contains invalid properties (CASSANDRA-5958)
+ * Improve leveled compaction's ability to find non-overlapping L0 compactions
+ to work on concurrently (CASSANDRA-5921)
+ * Notify indexer of columns shadowed by range tombstones (CASSANDRA-5614)
+ * Log Merkle tree stats (CASSANDRA-2698)
+ * Switch from crc32 to adler32 for compressed sstable checksums (CASSANDRA-5862)
+ * Improve offheap memcpy performance (CASSANDRA-5884)
+ * Use a range aware scanner for cleanup (CASSANDRA-2524)
+ * Cleanup doesn't need to inspect sstables that contain only local data
+ (CASSANDRA-5722)
+ * Add ability for CQL3 to list partition keys (CASSANDRA-4536)
+ * Improve native protocol serialization (CASSANDRA-5664)
+ * Upgrade Thrift to 0.9.1 (CASSANDRA-5923)
+ * Require superuser status for adding triggers (CASSANDRA-5963)
+ * Make standalone scrubber handle old and new style leveled manifest
+ (CASSANDRA-6005)
+ * Fix paxos bugs (CASSANDRA-6012, 6013, 6023)
+ * Fix paged ranges with multiple replicas (CASSANDRA-6004)
+ * Fix potential AssertionError during tracing (CASSANDRA-6041)
+ * Fix NPE in sstablesplit (CASSANDRA-6027)
+ * Migrate pre-2.0 key/value/column aliases to system.schema_columns
+ (CASSANDRA-6009)
+ * Paging filter empty rows too agressively (CASSANDRA-6040)
+ * Support variadic parameters for IN clauses (CASSANDRA-4210)
+ * cqlsh: return the result of CAS writes (CASSANDRA-5796)
+ * Fix validation of IN clauses with 2ndary indexes (CASSANDRA-6050)
+ * Support named bind variables in CQL (CASSANDRA-6033)
+Merged from 1.2:
+ * Allow cache-keys-to-save to be set at runtime (CASSANDRA-5980)
* Avoid second-guessing out-of-space state (CASSANDRA-5605)
* Tuning knobs for dealing with large blobs and many CFs (CASSANDRA-5982)
* (Hadoop) Fix CQLRW for thrift tables (CASSANDRA-6002)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/conf/cassandra.yaml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/Config.java
index 392e8c3,292161b..131c9ef
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@@ -177,8 -173,11 +177,10 @@@ public class Confi
public String memtable_allocator = "SlabAllocator";
- private static boolean loadYaml = true;
private static boolean outboundBindAny = false;
+ public volatile int tombstone_debug_threshold = 10000;
+
public static boolean getOutboundBindAny()
{
return outboundBindAny;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 1ff4832,a41c157..ec9570a
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -28,11 -28,9 +28,12 @@@ import java.util.concurrent.atomic.Atom
import java.util.regex.Pattern;
import javax.management.*;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
import com.google.common.collect.*;
import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.Uninterruptibles;
+ import org.cliffc.high_scale_lib.NonBlockingHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -48,12 -45,11 +49,8 @@@ import org.apache.cassandra.config.Sche
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
-import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
-import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.compaction.*;
- import org.apache.cassandra.db.filter.ColumnSlice;
- import org.apache.cassandra.db.filter.ExtendedFilter;
- import org.apache.cassandra.db.filter.IDiskAtomFilter;
- import org.apache.cassandra.db.filter.QueryFilter;
- import org.apache.cassandra.db.filter.SliceQueryFilter;
+ import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.db.marshal.AbstractType;
@@@ -1296,10 -1213,18 +1292,17 @@@ public class ColumnFamilyStore implemen
if (cf == null)
return null;
- // TODO this is necessary because when we collate supercolumns together, we don't check
- // their subcolumns for relevance, so we need to do a second prune post facto here.
- result = cf.isSuper() ? removeDeleted(cf, gcBefore) : removeDeletedCF(cf, gcBefore);
-
+ result = removeDeletedCF(cf, gcBefore);
}
+ removeDroppedColumns(result);
++
+ if (filter.filter instanceof SliceQueryFilter)
+ {
+ // Log the number of tombstones scanned on single key queries
+ metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored());
+ metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive());
+ }
}
finally
{
@@@ -2164,15 -1901,32 +2167,31 @@@
private void validateCompactionThresholds(int minThreshold, int maxThreshold)
{
- if (minThreshold > maxThreshold && maxThreshold != 0)
+ if (minThreshold > maxThreshold)
throw new RuntimeException(String.format("The min_compaction_threshold cannot be larger than the max_compaction_threshold. " +
"Min is '%d', Max is '%d'.", minThreshold, maxThreshold));
- }
- public boolean isCompactionDisabled()
- {
- return getMinimumCompactionThreshold() <= 0 || getMaximumCompactionThreshold() <= 0;
+ if (maxThreshold == 0 || minThreshold == 0)
+ throw new RuntimeException("Disabling compaction by setting min_compaction_threshold or max_compaction_threshold to 0 " +
+ "is deprecated, set the compaction strategy option 'enabled' to 'false' instead or use the nodetool command 'disableautocompaction'.");
}
+ public long getTombstonesPerLastRead()
+ {
+ return metric.tombstoneScannedHistogram.count();
+ }
+
+ public float getPercentageTombstonesPerLastRead()
+ {
+ long total = metric.tombstoneScannedHistogram.count() + metric.liveScannedHistogram.count();
+ return (metric.tombstoneScannedHistogram.count() / total);
+ }
+
+ public long getLiveCellsPerLastRead()
+ {
+ return metric.liveScannedHistogram.count();
+ }
+
// End JMX get/set.
public long estimateKeys()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index e810dc6,f937032..6105714
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@@ -281,8 -281,20 +281,17 @@@ public interface ColumnFamilyStoreMBea
*/
public void setCrcCheckChance(double crcCheckChance);
- /**
- * Disable automatic compaction.
- */
- public void disableAutoCompaction();
+ public boolean isAutoCompactionDisabled();
+ /** Number of tombstoned cells retreived during the last slicequery */
+ public long getTombstonesPerLastRead();
+
+ /** Percentage of tombstoned cells retreived during the last slicequery */
+ public float getPercentageTombstonesPerLastRead();
+
+ /** Number of live cells retreived during the last slicequery */
+ public long getLiveCellsPerLastRead();
+
public long estimateKeys();
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 11d7438,7e1ec6b..443ff8e
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@@ -21,17 -21,16 +21,13 @@@ import java.io.DataInput
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
- import java.util.Arrays;
- import java.util.ArrayList;
- import java.util.Collection;
- import java.util.Comparator;
- import java.util.Iterator;
- import java.util.List;
+ import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.ISSTableColumnIterator;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.columniterator.SSTableSliceIterator;
import org.apache.cassandra.db.marshal.AbstractType;
@@@ -202,48 -175,62 +198,50 @@@ public class SliceQueryFilter implement
}
Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
+ if (columnCounter.ignored() > DatabaseDescriptor.getTombstoneDebugThreshold())
+ logger.debug("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
}
- public int getLiveCount(ColumnFamily cf)
+ public int getLiveCount(ColumnFamily cf, long now)
{
- ColumnCounter counter = getColumnCounter(cf);
- for (IColumn column : cf)
- counter.count(column, cf);
- return counter.live();
+ return columnCounter(cf.getComparator(), now).countAll(cf).live();
}
- private ColumnCounter getColumnCounter(IColumnContainer container)
+ public ColumnCounter columnCounter(AbstractType<?> comparator, long now)
{
- AbstractType<?> comparator = container.getComparator();
if (compositesToGroup < 0)
- return new ColumnCounter();
+ return new ColumnCounter(now);
else if (compositesToGroup == 0)
- return new ColumnCounter.GroupByPrefix(null, 0);
+ return new ColumnCounter.GroupByPrefix(now, null, 0);
else
- return new ColumnCounter.GroupByPrefix((CompositeType)comparator, compositesToGroup);
+ return new ColumnCounter.GroupByPrefix(now, (CompositeType)comparator, compositesToGroup);
}
- public void trim(ColumnFamily cf, int trimTo)
+ public void trim(ColumnFamily cf, int trimTo, long now)
{
- ColumnCounter counter = getColumnCounter(cf);
+ ColumnCounter counter = columnCounter(cf.getComparator(), now);
- Collection<ByteBuffer> toRemove = null;
- boolean trimRemaining = false;
+ Collection<Column> columns = reversed
+ ? cf.getReverseSortedColumns()
+ : cf.getSortedColumns();
- Collection<IColumn> columns = reversed
- ? cf.getReverseSortedColumns()
- : cf.getSortedColumns();
+ DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(reversed);
- for (IColumn column : columns)
+ for (Iterator<Column> iter = columns.iterator(); iter.hasNext(); )
{
- if (trimRemaining)
- {
- toRemove.add(column.name());
- continue;
- }
+ Column column = iter.next();
+ counter.count(column, tester);
- counter.count(column, cf);
if (counter.live() > trimTo)
{
- toRemove = new HashSet<ByteBuffer>();
- toRemove.add(column.name());
- trimRemaining = true;
+ iter.remove();
+ while (iter.hasNext())
+ {
+ iter.next();
+ iter.remove();
+ }
}
}
-
- if (toRemove != null)
- {
- for (ByteBuffer columnName : toRemove)
- cf.remove(columnName);
- }
}
public ByteBuffer start()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index 0f9f87d,9bd90a9..b63bbfb
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@@ -334,7 -337,8 +340,9 @@@ public class ColumnFamilyMetric
Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed"));
Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("SpeculativeRetry"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram"));
+ Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram"));
}
class ColumnFamilyMetricNameFactory implements MetricNameFactory
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/service/StorageServiceMBean.java
----------------------------------------------------------------------