You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jm...@apache.org on 2021/10/13 13:31:06 UTC
[cassandra] branch trunk updated: Add logging and metrics for index
summary redistribution
This is an automated email from the ASF dual-hosted git repository.
jmckenzie pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2ce9b13 Add logging and metrics for index summary redistribution
2ce9b13 is described below
commit 2ce9b13e8ff1d5d54d87e403a25561a82c9490a0
Author: Marcus Eriksson <ma...@apache.org>
AuthorDate: Tue Oct 12 12:28:35 2021 -0400
Add logging and metrics for index summary redistribution
Patch by Marcus Eriksson, reviewed by David Capwell for CASSANDRA-17036
Co-authored-by: Marcus Eriksson <ma...@apache.org>
Co-authored-by: Josh McKenzie <jm...@apache.org>
---
CHANGES.txt | 1 +
.../org/apache/cassandra/io/sstable/IndexSummaryManager.java | 4 +++-
.../cassandra/io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
.../apache/cassandra/io/sstable/format/SSTableReader.java | 2 +-
src/java/org/apache/cassandra/metrics/CompactionMetrics.java | 5 ++++-
5 files changed, 17 insertions(+), 7 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 384576b..0c3dd81 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.1
+ * Add metrics and logging around index summary redistribution (CASSANDRA-17036)
* Add configuration options for minimum allowable replication factor and default replication factor (CASSANDRA-14557)
* Expose information about stored hints via a nodetool command and a virtual table (CASSANDRA-14795)
* Add broadcast_rpc_address to system.local (CASSANDRA-11181)
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
index aac0be4..ee6f349 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
@@ -30,6 +30,8 @@ import java.util.concurrent.TimeoutException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
+
+import com.codahale.metrics.Timer;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -231,7 +233,7 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean
Pair<Long, Map<TableId, LifecycleTransaction>> redistributionTransactionInfo = getRestributionTransactions();
Map<TableId, LifecycleTransaction> transactions = redistributionTransactionInfo.right;
long nonRedistributingOffHeapSize = redistributionTransactionInfo.left;
- try
+ try (Timer.Context ctx = CompactionManager.instance.getMetrics().indexSummaryRedistributionTime.time())
{
redistributeSummaries(new IndexSummaryRedistribution(transactions,
nonRedistributingOffHeapSize,
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
index 90a8621..bac3280 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
@@ -46,6 +47,7 @@ import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
public class IndexSummaryRedistribution extends CompactionInfo.Holder
{
@@ -80,6 +82,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
public List<SSTableReader> redistributeSummaries() throws IOException
{
+ long start = nanoTime();
logger.info("Redistributing index summaries");
List<SSTableReader> redistribute = new ArrayList<>();
for (LifecycleTransaction txn : transactions.values())
@@ -91,7 +94,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
for (SSTableReader sstable : redistribute)
total += sstable.getIndexSummaryOffHeapSize();
- logger.trace("Beginning redistribution of index summaries for {} sstables with memory pool size {} MB; current spaced used is {} MB",
+ logger.info("Beginning redistribution of index summaries for {} sstables with memory pool size {} MB; current spaced used is {} MB",
redistribute.size(), memoryPoolBytes / 1024L / 1024L, total / 1024.0 / 1024.0);
final Map<SSTableReader, Double> readRates = new HashMap<>(redistribute.size());
@@ -129,9 +132,9 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
total = nonRedistributingOffHeapSize;
for (SSTableReader sstable : newSSTables)
total += sstable.getIndexSummaryOffHeapSize();
- if (logger.isTraceEnabled())
- logger.trace("Completed resizing of index summaries; current approximate memory used: {}",
- FBUtilities.prettyPrintMemory(total));
+
+ logger.info("Completed resizing of index summaries; current approximate memory used: {} MB, time spent: {}ms",
+ total / 1024.0 / 1024.0, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start));
return newSSTables;
}
@@ -243,6 +246,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
}
// downsample first, then upsample
+ logger.info("index summaries: downsample: {}, force resample: {}, upsample: {}, force upsample: {}", toDownsample.size(), forceResample.size(), toUpsample.size(), forceUpsample.size());
toDownsample.addAll(forceResample);
toDownsample.addAll(toUpsample);
toDownsample.addAll(forceUpsample);
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index 4af5743..771974d 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -742,7 +742,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
}
catch (IOException e)
{
- logger.trace("Cannot save SSTable Summary: ", e);
+ logger.error("Cannot save SSTable Summary: ", e);
// corrupted hence delete it and let it load it now.
if (summariesFile.exists())
diff --git a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
index 375adc4..0fe1ec7 100644
--- a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java
@@ -23,6 +23,7 @@ import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
@@ -51,7 +52,8 @@ public class CompactionMetrics
public final Meter totalCompactionsCompleted;
/** Total number of bytes compacted since server [re]start */
public final Counter bytesCompacted;
-
+ /** Time spent redistributing index summaries */
+ public final Timer indexSummaryRedistributionTime;
/** Total number of compactions that have had sstables drop out of them */
public final Counter compactionsReduced;
@@ -149,5 +151,6 @@ public class CompactionMetrics
compactionsReduced = Metrics.counter(factory.createMetricName("CompactionsReduced"));
sstablesDropppedFromCompactions = Metrics.counter(factory.createMetricName("SSTablesDroppedFromCompaction"));
compactionsAborted = Metrics.counter(factory.createMetricName("CompactionsAborted"));
+ indexSummaryRedistributionTime = Metrics.timer(factory.createMetricName("IndexSummaryRedistributionTime"));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org