You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2015/07/02 20:58:34 UTC
[6/8] cassandra git commit: Warn when an extra-large partition is
compacted, CASSANDRA-9643
Warn when an extra-large partition is compacted, CASSANDRA-9643
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/492c65a6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/492c65a6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/492c65a6
Branch: refs/heads/cassandra-2.2
Commit: 492c65a677c02727c6186be801eb81763b7a312d
Parents: b19dd38
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Tue Jun 30 17:12:56 2015 +0800
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Jul 2 11:57:30 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 3 +++
src/java/org/apache/cassandra/config/Config.java | 1 +
.../apache/cassandra/config/DatabaseDescriptor.java | 2 ++
.../io/sstable/format/big/BigTableWriter.java | 15 ++++++++++++++-
5 files changed, 21 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0b38ff0..4042722 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.0-rc2
+ * Warn when an extra-large partition is compacted (CASSANDRA-9643)
* (cqlsh) Allow setting the initial connection timeout (CASSANDRA-9601)
* BulkLoader has --transport-factory option but does not use it (CASSANDRA-9675)
* Allow JMX over SSL directly from nodetool (CASSANDRA-9090)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index dba8e1f..640af8f 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -622,6 +622,9 @@ batch_size_fail_threshold_in_kb: 50
# of compaction, including validation compaction.
compaction_throughput_mb_per_sec: 16
+# Log a warning when compacting partitions larger than this value
+compaction_large_partition_warning_threshold_mb: 100
+
# When compacting, the replacement sstable(s) can be opened before they
# are completely written, and used in place of the prior sstables for
# any range that has been written. This helps to smoothly transfer reads
http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 5beeef2..9f37d9a 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -152,6 +152,7 @@ public class Config
public volatile int batch_size_fail_threshold_in_kb = 50;
public Integer concurrent_compactors;
public volatile Integer compaction_throughput_mb_per_sec = 16;
+ public volatile Integer compaction_large_partition_warning_threshold_mb = 100;
public Integer max_streaming_retries = 3;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 39a06cb..632bf0a 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1054,6 +1054,8 @@ public class DatabaseDescriptor
conf.compaction_throughput_mb_per_sec = value;
}
+ public static int getCompactionLargePartitionWarningThreshold() { return conf.compaction_large_partition_warning_threshold_mb * 1024 * 1024; }
+
public static boolean getDisableSTCSInL0()
{
return Boolean.getBoolean("cassandra.disable_stcs_in_l0");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 30b55a0..c3caa0f 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -137,7 +137,9 @@ public class BigTableWriter extends SSTableWriter
if (entry == null)
return null;
long endPosition = dataFile.getFilePointer();
- metadataCollector.update(endPosition - startPosition, row.columnStats());
+ long rowSize = endPosition - startPosition;
+ maybeLogLargePartitionWarning(row.key, rowSize);
+ metadataCollector.update(rowSize, row.columnStats());
afterAppend(row.key, endPosition, entry);
return entry;
}
@@ -169,9 +171,20 @@ public class BigTableWriter extends SSTableWriter
{
throw new FSWriteError(e, dataFile.getPath());
}
+ long rowSize = endPosition - startPosition;
+ maybeLogLargePartitionWarning(decoratedKey, rowSize);
metadataCollector.update(endPosition - startPosition, cf.getColumnStats());
}
+ private void maybeLogLargePartitionWarning(DecoratedKey key, long rowSize)
+ {
+ if (rowSize > DatabaseDescriptor.getCompactionLargePartitionWarningThreshold())
+ {
+ String keyString = metadata.getKeyValidator().getString(key.getKey());
+ logger.warn("Compacting large partition {}/{}:{} ({} bytes)", metadata.ksName, metadata.cfName, keyString, rowSize);
+ }
+ }
+
private static RowIndexEntry rawAppend(ColumnFamily cf, long startPosition, DecoratedKey key, DataOutputPlus out) throws IOException
{
assert cf.hasColumns() || cf.isMarkedForDelete();