You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2015/07/02 20:58:30 UTC
[2/8] cassandra git commit: Warn when an extra-large partition is
compacted, CASSANDRA-9643
Warn when an extra-large partition is compacted, CASSANDRA-9643
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/921eb7e8
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/921eb7e8
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/921eb7e8
Branch: refs/heads/trunk
Commit: 921eb7e8065221be586f63c64f9b1de582136ca5
Parents: 2595e74
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Tue Jun 30 17:12:56 2015 +0800
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Jul 2 11:37:50 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 3 +++
src/java/org/apache/cassandra/config/Config.java | 1 +
.../cassandra/config/DatabaseDescriptor.java | 2 ++
.../apache/cassandra/io/sstable/SSTableWriter.java | 17 +++++++++++++++--
5 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c37ef02..78e23fb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.8
+ * Warn when an extra-large partition is compacted (CASSANDRA-9643)
* Eliminate strong self-reference chains in sstable ref tidiers (CASSANDRA-9656)
* Ensure StreamSession uses canonical sstable reader instances (CASSANDRA-9700)
* Ensure memtable book keeping is not corrupted in the event we shrink usage (CASSANDRA-9681)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 9d6d135..e4b7cbd 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -574,6 +574,9 @@ batch_size_warn_threshold_in_kb: 5
# of compaction, including validation compaction.
compaction_throughput_mb_per_sec: 16
+# Log a warning when compacting partitions larger than this value
+compaction_large_partition_warning_threshold_mb: 100
+
# When compacting, the replacement sstable(s) can be opened before they
# are completely written, and used in place of the prior sstables for
# any range that has been written. This helps to smoothly transfer reads
http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 201e729..31e4cd5 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -146,6 +146,7 @@ public class Config
public Integer batch_size_warn_threshold_in_kb = 5;
public Integer concurrent_compactors;
public volatile Integer compaction_throughput_mb_per_sec = 16;
+ public volatile Integer compaction_large_partition_warning_threshold_mb = 100;
public Integer max_streaming_retries = 3;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index a2a9e48..68dcd5c 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1103,6 +1103,8 @@ public class DatabaseDescriptor
conf.compaction_throughput_mb_per_sec = value;
}
+ public static int getCompactionLargePartitionWarningThreshold() { return conf.compaction_large_partition_warning_threshold_mb * 1024 * 1024; }
+
public static boolean getDisableSTCSInL0()
{
return Boolean.getBoolean("cassandra.disable_stcs_in_l0");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
index a39c134..8e0b5f7 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
@@ -198,7 +198,9 @@ public class SSTableWriter extends SSTable
throw new FSWriteError(e, dataFile.getPath());
}
long endPosition = dataFile.getFilePointer();
- sstableMetadataCollector.update(endPosition - startPosition, row.columnStats());
+ long rowSize = endPosition - startPosition;
+ maybeLogLargePartitionWarning(row.key, rowSize);
+ sstableMetadataCollector.update(rowSize, row.columnStats());
afterAppend(row.key, endPosition, entry);
return entry;
}
@@ -225,7 +227,18 @@ public class SSTableWriter extends SSTable
{
throw new FSWriteError(e, dataFile.getPath());
}
- sstableMetadataCollector.update(endPosition - startPosition, cf.getColumnStats());
+ long rowSize = endPosition - startPosition;
+ maybeLogLargePartitionWarning(decoratedKey, rowSize);
+ sstableMetadataCollector.update(rowSize, cf.getColumnStats());
+ }
+
+ private void maybeLogLargePartitionWarning(DecoratedKey key, long rowSize)
+ {
+ if (rowSize > DatabaseDescriptor.getCompactionLargePartitionWarningThreshold())
+ {
+ String keyString = metadata.getKeyValidator().getString(key.getKey());
+ logger.warn("Compacting large partition {}/{}:{} ({} bytes)", metadata.ksName, metadata.cfName, keyString, rowSize);
+ }
}
public static RowIndexEntry rawAppend(ColumnFamily cf, long startPosition, DecoratedKey key, DataOutputPlus out) throws IOException