You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2016/08/23 14:50:29 UTC
[2/5] cassandra git commit: Calculate last compacted keys from higher
level
Calculate last compacted keys from higher level
patch by Dikang Gu; reviewed by marcuse for CASSANDRA-6216
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6808e755
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6808e755
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6808e755
Branch: refs/heads/trunk
Commit: 6808e7555762236b6222f65bf066f90f7597405e
Parents: d0fc953
Author: Dikang Gu <di...@gmail.com>
Authored: Sat Aug 20 00:55:22 2016 -0700
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 23 16:18:39 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../compaction/LeveledCompactionStrategy.java | 7 ++++
.../db/compaction/LeveledManifest.java | 36 ++++++++++++++++++++
3 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d384fb3..0d461f3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.9
+ * Calculate last compacted key on startup (CASSANDRA-6216)
* Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190)
* Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
* Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index cd74620..65fca27 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@ -85,6 +85,13 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
return manifest.getAllLevelSize();
}
+ @Override
+ public void startup()
+ {
+ manifest.calculateLastCompactedKeys();
+ super.startup();
+ }
+
/**
* the only difference between background and maximal in LCS is that maximal is still allowed
* (by explicit user request) even when compaction is disabled.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 2bfe88f..cf3ae1a 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -30,6 +30,7 @@ import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,6 +67,7 @@ public class LeveledManifest
@VisibleForTesting
protected final List<SSTableReader>[] generations;
private final PartitionPosition[] lastCompactedKeys;
+ private volatile boolean calculatedLastCompactedKeys = false;
private final long maxSSTableSizeInBytes;
private final SizeTieredCompactionStrategyOptions options;
private final int [] compactionCounter;
@@ -104,9 +106,38 @@ public class LeveledManifest
{
manifest.repairOverlappingSSTables(i);
}
+ manifest.calculateLastCompactedKeys();
return manifest;
}
+ /**
+ * If we want to start compaction in level n, find the newest (by modification time) file in level n+1
+ * and use its last token for last compacted key in level n;
+ */
+ public void calculateLastCompactedKeys()
+ {
+ for (int i = 0; i < generations.length - 1; i++)
+ {
+ // this level is empty
+ if (generations[i + 1].isEmpty())
+ continue;
+
+ SSTableReader sstableWithMaxModificationTime = null;
+ long maxModificationTime = Long.MIN_VALUE;
+ for (SSTableReader ssTableReader : generations[i + 1])
+ {
+ long modificationTime = ssTableReader.getCreationTimeFor(Component.DATA);
+ if (modificationTime >= maxModificationTime)
+ {
+ sstableWithMaxModificationTime = ssTableReader;
+ maxModificationTime = modificationTime;
+ }
+ }
+
+ lastCompactedKeys[i] = sstableWithMaxModificationTime.last;
+ }
+ }
+
public synchronized void add(SSTableReader reader)
{
int level = reader.getSSTableLevel();
@@ -307,6 +338,11 @@ public class LeveledManifest
// This isn't a magic wand -- if you are consistently writing too fast for LCS to keep
// up, you're still screwed. But if instead you have intermittent bursts of activity,
// it can help a lot.
+ if (!calculatedLastCompactedKeys)
+ {
+ this.calculateLastCompactedKeys();
+ }
+
for (int i = generations.length - 1; i > 0; i--)
{
List<SSTableReader> sstables = getLevel(i);