You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2021/07/02 08:08:00 UTC
[cassandra] 01/02: Display bytes per level in tablestats for LCS
tables
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 3f6f7be63faf2d07b274caf25a2fa1dcbde14187
Author: Marcus Eriksson <ma...@apache.org>
AuthorDate: Wed Jun 30 16:34:09 2021 +0200
Display bytes per level in tablestats for LCS tables
Patch by marcuse; reviewed by Brandon Williams for CASSANDRA-16779
---
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 5 +++
.../cassandra/db/ColumnFamilyStoreMBean.java | 6 +++
.../db/compaction/CompactionStrategyManager.java | 38 +++++++++++++++++
.../db/compaction/LeveledCompactionStrategy.java | 5 +++
.../db/compaction/LeveledGenerations.java | 8 ++++
.../cassandra/db/compaction/LeveledManifest.java | 5 +++
.../cassandra/tools/nodetool/stats/StatsTable.java | 1 +
.../tools/nodetool/stats/TableStatsHolder.java | 12 ++++++
.../tools/nodetool/stats/TableStatsPrinter.java | 6 ++-
.../compaction/LeveledCompactionStrategyTest.java | 47 ++++++++++++++++++++++
11 files changed, 133 insertions(+), 1 deletion(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index dcfacb4..d2a0cff 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.1
+ * Display bytes per level in tablestats for LCS tables (CASSANDRA-16799)
* Add isolated flush timer to CommitLogMetrics and ensure writes correspond to single WaitingOnCommit data points (CASSANDRA-16701)
* Add a system property to set hostId if not yet initialized (CASSANDRA-14582)
* GossiperTest.testHasVersion3Nodes didn't take into account trunk version changes, fixed to rely on latest version (CASSANDRA-16651)
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 113a916..df9f763 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -2638,6 +2638,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return compactionStrategyManager.getSSTableCountPerLevel();
}
+ public long[] getPerLevelSizeBytes()
+ {
+ return compactionStrategyManager.getPerLevelSizeBytes();
+ }
+
public int getLevelFanoutSize()
{
return compactionStrategyManager.getLevelFanoutSize();
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index 0360e34..f06b65a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -203,6 +203,12 @@ public interface ColumnFamilyStoreMBean
public int[] getSSTableCountPerLevel();
/**
+ * @return total size on disk for each level. null unless leveled compaction is used.
+ * array index corresponds to level(int[0] is for level 0, ...).
+ */
+ public long[] getPerLevelSizeBytes();
+
+ /**
* @return sstable fanout size for level compaction strategy.
*/
public int getLevelFanoutSize();
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
index deece30..d7d3ba5 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
@@ -587,6 +587,29 @@ public class CompactionStrategyManager implements INotificationConsumer
return null;
}
+ public long[] getPerLevelSizeBytes()
+ {
+ readLock.lock();
+ try
+ {
+ if (repaired.first() instanceof LeveledCompactionStrategy)
+ {
+ long [] res = new long[LeveledGenerations.MAX_LEVEL_COUNT];
+ for (AbstractCompactionStrategy strategy : getAllStrategies())
+ {
+ long[] repairedCountPerLevel = ((LeveledCompactionStrategy) strategy).getAllLevelSizeBytes();
+ res = sumArrays(res, repairedCountPerLevel);
+ }
+ return res;
+ }
+ return null;
+ }
+ finally
+ {
+ readLock.unlock();
+ }
+ }
+
static int[] sumArrays(int[] a, int[] b)
{
int[] res = new int[Math.max(a.length, b.length)];
@@ -602,6 +625,21 @@ public class CompactionStrategyManager implements INotificationConsumer
return res;
}
+ static long[] sumArrays(long[] a, long[] b)
+ {
+ long[] res = new long[Math.max(a.length, b.length)];
+ for (int i = 0; i < res.length; i++)
+ {
+ if (i < a.length && i < b.length)
+ res[i] = a[i] + b[i];
+ else if (i < a.length)
+ res[i] = a[i];
+ else
+ res[i] = b[i];
+ }
+ return res;
+ }
+
/**
* Should only be called holding the readLock
*/
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index dd7c9df..6faca63 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@ -109,6 +109,11 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
return manifest.getAllLevelSize();
}
+ public long[] getAllLevelSizeBytes()
+ {
+ return manifest.getAllLevelSizeBytes();
+ }
+
@Override
public void startup()
{
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java b/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
index 64027f2..2cc1b1a 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
@@ -226,6 +226,14 @@ class LeveledGenerations
return counts;
}
+ long[] getAllLevelSizeBytes()
+ {
+ long[] sums = new long[levelCount()];
+ for (int i = 0; i < sums.length; i++)
+ sums[i] = get(i).stream().map(SSTableReader::onDiskLength).reduce(0L, Long::sum);
+ return sums;
+ }
+
Set<SSTableReader> allSSTables()
{
ImmutableSet.Builder<SSTableReader> builder = ImmutableSet.builder();
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 7c865c7..8b636fc 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -395,6 +395,11 @@ public class LeveledManifest
return generations.getAllLevelSize();
}
+ public synchronized long[] getAllLevelSizeBytes()
+ {
+ return generations.getAllLevelSizeBytes();
+ }
+
@VisibleForTesting
public synchronized int remove(SSTableReader reader)
{
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java
index d897eab..6f5d904 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java
@@ -69,5 +69,6 @@ public class StatsTable
public long maximumTombstonesPerSliceLastFiveMinutes;
public String droppedMutations;
public List<String> sstablesInEachLevel = new ArrayList<>();
+ public List<String> sstableBytesInEachLevel = new ArrayList<>();
public Boolean isInCorrectLocation = null; // null: option not active
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
index b1685e6..b66c802 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java
@@ -118,6 +118,7 @@ public class TableStatsHolder implements StatsHolder
{
Map<String, Object> mpTable = new HashMap<>();
mpTable.put("sstables_in_each_level", table.sstablesInEachLevel);
+ mpTable.put("sstable_bytes_in_each_level", table.sstableBytesInEachLevel);
mpTable.put("space_used_live", table.spaceUsedLive);
mpTable.put("space_used_total", table.spaceUsedTotal);
mpTable.put("space_used_by_snapshots_total", table.spaceUsedBySnapshotsTotal);
@@ -235,6 +236,17 @@ public class TableStatsHolder implements StatsHolder
}
}
+ long[] leveledSSTablesBytes = table.getPerLevelSizeBytes();
+ if (leveledSSTablesBytes != null)
+ {
+ statsTable.isLeveledSstable = true;
+ for (int level = 0; level < leveledSSTablesBytes.length; level++)
+ {
+ long size = leveledSSTablesBytes[level];
+ statsTable.sstableBytesInEachLevel.add(format(size, humanReadable));
+ }
+ }
+
if (locationCheck)
statsTable.isInCorrectLocation = !table.hasMisplacedSSTables();
diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java
index f4188e4..50a22d6 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java
@@ -79,8 +79,12 @@ public class TableStatsPrinter<T extends StatsHolder>
out.println(indent + "SSTable count: " + table.sstableCount);
out.println(indent + "Old SSTable count: " + table.oldSSTableCount);
if (table.isLeveledSstable)
+ {
out.println(indent + "SSTables in each level: [" + String.join(", ",
- table.sstablesInEachLevel) + "]");
+ table.sstablesInEachLevel) + "]");
+ out.println(indent + "SSTable bytes in each level: [" + String.join(", ",
+ table.sstableBytesInEachLevel) + "]");
+ }
out.println(indent + "Space used (live): " + table.spaceUsedLive);
out.println(indent + "Space used (total): " + table.spaceUsedTotal);
diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
index 2d6835b..68a9ac8 100644
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
@@ -771,6 +771,53 @@ public class LeveledCompactionStrategyTest
}
return newLevels;
}
+ @Test
+ public void testPerLevelSizeBytes() throws IOException
+ {
+ byte [] b = new byte[100];
+ new Random().nextBytes(b);
+ ByteBuffer value = ByteBuffer.wrap(b);
+ int rows = 5;
+ int columns = 5;
+
+ cfs.disableAutoCompaction();
+ for (int r = 0; r < rows; r++)
+ {
+ UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), String.valueOf(r));
+ for (int c = 0; c < columns; c++)
+ update.newRow("column" + c).add("val", value);
+ update.applyUnsafe();
+ }
+ cfs.forceBlockingFlush();
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ long [] levelSizes = cfs.getPerLevelSizeBytes();
+ for (int i = 0; i < levelSizes.length; i++)
+ {
+ if (i != 0)
+ assertEquals(0, levelSizes[i]);
+ else
+ assertEquals(sstable.onDiskLength(), levelSizes[i]);
+ }
+
+ assertEquals(sstable.onDiskLength(), cfs.getPerLevelSizeBytes()[0]);
+
+ LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) ( cfs.getCompactionStrategyManager()).getStrategies().get(1).get(0);
+ strategy.manifest.remove(sstable);
+ sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 2);
+ sstable.reloadSSTableMetadata();
+ strategy.manifest.addSSTables(Collections.singleton(sstable));
+
+ levelSizes = cfs.getPerLevelSizeBytes();
+ for (int i = 0; i < levelSizes.length; i++)
+ {
+ if (i != 2)
+ assertEquals(0, levelSizes[i]);
+ else
+ assertEquals(sstable.onDiskLength(), levelSizes[i]);
+ }
+
+ }
/**
* brute-force checks if the new sstables can be added to the correct level in manifest
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org