You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ja...@apache.org on 2017/01/19 12:30:36 UTC
cassandra git commit: Added compaction metrics to track failures when
running low on disk space
Repository: cassandra
Updated Branches:
refs/heads/trunk d32ae4a60 -> 52df6a58d
Added compaction metrics to track failures when running low on disk space
patch by John Haddad; reviewed by jasobrown for CASSANDRA-13015
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/52df6a58
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/52df6a58
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/52df6a58
Branch: refs/heads/trunk
Commit: 52df6a58d18200c11d2e9d88fc9276b5ef1ee06a
Parents: d32ae4a
Author: Jon Haddad <jo...@jonhaddad.com>
Authored: Tue Jan 10 22:32:34 2017 -0800
Committer: Jason Brown <ja...@gmail.com>
Committed: Thu Jan 19 04:29:44 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/compaction/CompactionManager.java | 16 ++++++++++++++++
.../cassandra/db/compaction/CompactionTask.java | 15 +++++++++++++--
.../apache/cassandra/metrics/CompactionMetrics.java | 15 +++++++++++++++
4 files changed, 45 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52df6a58/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a67cf43..8f68618 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.0
+ * Improved compactions metrics (CASSANDRA-13015)
* Speed-up start-up sequence by avoiding un-needed flushes (CASSANDRA-13031)
* Use Caffeine (W-TinyLFU) for on-heap caches (CASSANDRA-10855)
* Thrift removal (CASSANDRA-11115)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52df6a58/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index fb11bad..9c74f62 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -1838,6 +1838,22 @@ public class CompactionManager implements CompactionManagerMBean
void finishCompaction(CompactionInfo.Holder ci);
}
+ public void incrementAborted()
+ {
+ metrics.compactionsAborted.inc();
+ }
+
+ public void incrementCompactionsReduced()
+ {
+ metrics.compactionsReduced.inc();
+ }
+
+ public void incrementSstablesDropppedFromCompactions(long num)
+ {
+ metrics.sstablesDropppedFromCompactions.inc(num);
+ }
+
+
public List<Map<String, String>> getCompactions()
{
List<Holder> compactionHolders = CompactionMetrics.getCompactions();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52df6a58/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index b2e9b8c..62efa3d 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -27,7 +27,6 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
-import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.RateLimiter;
import org.apache.cassandra.db.Directories;
@@ -333,23 +332,35 @@ public class CompactionTask extends AbstractCompactionTask
}
CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
-
+ int sstablesRemoved = 0;
while(true)
{
long expectedWriteSize = cfs.getExpectedCompactedFileSize(transaction.originals(), compactionType);
long estimatedSSTables = Math.max(1, expectedWriteSize / strategy.getMaxSSTableBytes());
if(cfs.getDirectories().hasAvailableDiskSpace(estimatedSSTables, expectedWriteSize))
+ {
+ // we're ok now on space so now we track the failures, if any
+ if(sstablesRemoved > 0)
+ {
+ CompactionManager.instance.incrementCompactionsReduced();
+ CompactionManager.instance.incrementSstablesDropppedFromCompactions(sstablesRemoved);
+ }
+
break;
+ }
if (!reduceScopeForLimitedSpace(expectedWriteSize))
{
// we end up here if we can't take any more sstables out of the compaction.
// usually means we've run out of disk space
String msg = String.format("Not enough space for compaction, estimated sstables = %d, expected write size = %d", estimatedSSTables, expectedWriteSize);
+
logger.warn(msg);
+ CompactionManager.instance.incrementAborted();
throw new RuntimeException(msg);
}
+ sstablesRemoved++;
logger.warn("Not enough space for compaction, {}MB estimated. Reducing scope.",
(float) expectedWriteSize / 1024 / 1024);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/52df6a58/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
index 9aef0f8..96ca38c 100644
--- a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
@@ -55,6 +55,16 @@ public class CompactionMetrics implements CompactionManager.CompactionExecutorSt
/** Total number of bytes compacted since server [re]start */
public final Counter bytesCompacted;
+
+ /** Total number of compactions that have had sstables drop out of them */
+ public final Counter compactionsReduced;
+
+ /** Total number of sstables that have been dropped out */
+ public final Counter sstablesDropppedFromCompactions;
+
+ /** Total number of compactions which have outright failed due to lack of disk space */
+ public final Counter compactionsAborted;
+
public CompactionMetrics(final ThreadPoolExecutor... collectors)
{
pendingTasks = Metrics.register(factory.createMetricName("PendingTasks"), new Gauge<Integer>()
@@ -137,6 +147,11 @@ public class CompactionMetrics implements CompactionManager.CompactionExecutorSt
});
totalCompactionsCompleted = Metrics.meter(factory.createMetricName("TotalCompactionsCompleted"));
bytesCompacted = Metrics.counter(factory.createMetricName("BytesCompacted"));
+
+ // compaction failure metrics
+ compactionsReduced = Metrics.counter(factory.createMetricName("CompactionsReduced"));
+ sstablesDropppedFromCompactions = Metrics.counter(factory.createMetricName("SSTablesDroppedFromCompaction"));
+ compactionsAborted = Metrics.counter(factory.createMetricName("CompactionsAborted"));
}
public void beginCompaction(CompactionInfo.Holder ci)