You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2018/02/08 07:42:57 UTC
cassandra git commit: Make it possible to change neverPurgeTombstones
during runtime
Repository: cassandra
Updated Branches:
refs/heads/trunk d0b34d383 -> 0f58f6c65
Make it possible to change neverPurgeTombstones during runtime
Patch by marcuse; reviewed by Chris Lohfink for CASSANDRA-14214
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0f58f6c6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0f58f6c6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0f58f6c6
Branch: refs/heads/trunk
Commit: 0f58f6c6501518e57ba021cb959b288fd533f472
Parents: d0b34d3
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Feb 5 09:27:52 2018 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 8 08:41:39 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 19 +++++
.../cassandra/db/ColumnFamilyStoreMBean.java | 4 +
.../compaction/AbstractCompactionStrategy.java | 2 +-
.../db/compaction/CompactionController.java | 16 ++--
.../db/compaction/CompactionsCQLTest.java | 83 ++++++++++++++++++++
6 files changed, 119 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cadd4c8..191caaf 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.0
+ * Make it possible to change neverPurgeTombstones during runtime (CASSANDRA-14214)
* Remove GossipDigestSynVerbHandler#doSort() (CASSANDRA-14174)
* Add nodetool clientlist (CASSANDRA-13665)
* Revert ProtocolVersion changes from CASSANDRA-7544 (CASSANDRA-14211)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index c12b474..8e7b220 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -218,6 +218,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
@VisibleForTesting
final DiskBoundaryManager diskBoundaryManager = new DiskBoundaryManager();
+ private volatile boolean neverPurgeTombstones = false;
+
public static void shutdownPostFlushExecutor() throws InterruptedException
{
postFlushExecutor.shutdown();
@@ -2636,4 +2638,21 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
diskBoundaryManager.invalidate();
}
+
+ @Override
+ public void setNeverPurgeTombstones(boolean value)
+ {
+ if (neverPurgeTombstones != value)
+ logger.info("Changing neverPurgeTombstones for {}.{} from {} to {}", keyspace.getName(), getTableName(), neverPurgeTombstones, value);
+ else
+ logger.info("Not changing neverPurgeTombstones for {}.{}, it is {}", keyspace.getName(), getTableName(), neverPurgeTombstones);
+
+ neverPurgeTombstones = value;
+ }
+
+ @Override
+ public boolean getNeverPurgeTombstones()
+ {
+ return neverPurgeTombstones;
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index e361ffe..bdb842d 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -195,4 +195,8 @@ public interface ColumnFamilyStoreMBean
Enable/Disable compaction space check
*/
public void compactionDiskSpaceCheck(boolean enable);
+
+ public void setNeverPurgeTombstones(boolean value);
+
+ public boolean getNeverPurgeTombstones();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index e88524f..a43761f 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -350,7 +350,7 @@ public abstract class AbstractCompactionStrategy
*/
protected boolean worthDroppingTombstones(SSTableReader sstable, int gcBefore)
{
- if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES)
+ if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones())
return false;
// since we use estimations to calculate, there is a chance that compaction will not drop tombstones actually.
// if that happens we will end up in infinite compaction loop, so first we check enough if enough time has
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/src/java/org/apache/cassandra/db/compaction/CompactionController.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
index 32ce67a..59bba0a 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
@@ -112,6 +112,12 @@ public class CompactionController implements AutoCloseable
return;
}
+ if (cfs.getNeverPurgeTombstones())
+ {
+ logger.debug("not refreshing overlaps for {}.{} - neverPurgeTombstones is enabled", cfs.keyspace.getName(), cfs.getTableName());
+ return;
+ }
+
for (SSTableReader reader : overlappingSSTables)
{
if (reader.isMarkedCompacted())
@@ -124,7 +130,7 @@ public class CompactionController implements AutoCloseable
private void refreshOverlaps()
{
- if (NEVER_PURGE_TOMBSTONES)
+ if (NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones())
return;
if (this.overlappingSSTables != null)
@@ -167,8 +173,8 @@ public class CompactionController implements AutoCloseable
{
logger.trace("Checking droppable sstables in {}", cfStore);
- if (NEVER_PURGE_TOMBSTONES || compacting == null)
- return Collections.emptySet();
+ if (NEVER_PURGE_TOMBSTONES || compacting == null || cfStore.getNeverPurgeTombstones())
+ return Collections.<SSTableReader>emptySet();
if (cfStore.getCompactionStrategyManager().onlyPurgeRepairedTombstones() && !Iterables.all(compacting, SSTableReader::isRepaired))
return Collections.emptySet();
@@ -259,7 +265,7 @@ public class CompactionController implements AutoCloseable
*/
public LongPredicate getPurgeEvaluator(DecoratedKey key)
{
- if (NEVER_PURGE_TOMBSTONES || !compactingRepaired())
+ if (NEVER_PURGE_TOMBSTONES || !compactingRepaired() || cfs.getNeverPurgeTombstones())
return time -> false;
overlapIterator.update(key);
@@ -321,7 +327,7 @@ public class CompactionController implements AutoCloseable
// caller must close iterators
public Iterable<UnfilteredRowIterator> shadowSources(DecoratedKey key, boolean tombstoneOnly)
{
- if (!provideTombstoneSources() || !compactingRepaired() || NEVER_PURGE_TOMBSTONES)
+ if (!provideTombstoneSources() || !compactingRepaired() || NEVER_PURGE_TOMBSTONES || cfs.getNeverPurgeTombstones())
return null;
overlapIterator.update(key);
return Iterables.filter(Iterables.transform(overlapIterator.overlaps(),
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f58f6c6/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
index 7873ac9..4d5215e 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
@@ -25,7 +25,14 @@ import org.junit.Test;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -226,6 +233,82 @@ public class CompactionsCQLTest extends CQLTester
getCurrentColumnFamilyStore().setCompactionParameters(localOptions);
}
+ @Test
+ public void testPerCFSNeverPurgeTombstonesCell() throws Throwable
+ {
+ testPerCFSNeverPurgeTombstonesHelper(true);
+ }
+
+ @Test
+ public void testPerCFSNeverPurgeTombstones() throws Throwable
+ {
+ testPerCFSNeverPurgeTombstonesHelper(false);
+ }
+
+
+ public void testPerCFSNeverPurgeTombstonesHelper(boolean deletedCell) throws Throwable
+ {
+ createTable("CREATE TABLE %s (id int primary key, b text) with gc_grace_seconds = 0");
+ for (int i = 0; i < 100; i++)
+ {
+ execute("INSERT INTO %s (id, b) VALUES (?, ?)", i, String.valueOf(i));
+ }
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+
+ assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false);
+ if (deletedCell)
+ execute("UPDATE %s SET b=null WHERE id = ?", 50);
+ else
+ execute("DELETE FROM %s WHERE id = ?", 50);
+ getCurrentColumnFamilyStore().setNeverPurgeTombstones(false);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Thread.sleep(2000); // wait for gcgs to pass
+ getCurrentColumnFamilyStore().forceMajorCompaction();
+ assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false);
+ if (deletedCell)
+ execute("UPDATE %s SET b=null WHERE id = ?", 44);
+ else
+ execute("DELETE FROM %s WHERE id = ?", 44);
+ getCurrentColumnFamilyStore().setNeverPurgeTombstones(true);
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ Thread.sleep(1100);
+ getCurrentColumnFamilyStore().forceMajorCompaction();
+ assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), true);
+ // disable it again and make sure the tombstone is gone:
+ getCurrentColumnFamilyStore().setNeverPurgeTombstones(false);
+ getCurrentColumnFamilyStore().forceMajorCompaction();
+ assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false);
+ getCurrentColumnFamilyStore().truncateBlocking();
+ }
+
+ private void assertTombstones(SSTableReader sstable, boolean expectTS)
+ {
+ boolean foundTombstone = false;
+ try(ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext())
+ {
+ try (UnfilteredRowIterator iter = scanner.next())
+ {
+ if (!iter.partitionLevelDeletion().isLive())
+ foundTombstone = true;
+ while (iter.hasNext())
+ {
+ Unfiltered unfiltered = iter.next();
+ assertTrue(unfiltered instanceof Row);
+ for (Cell c : ((Row)unfiltered).cells())
+ {
+ if (c.isTombstone())
+ foundTombstone = true;
+ }
+
+ }
+ }
+ }
+ }
+ assertEquals(expectTS, foundTombstone);
+ }
+
public boolean verifyStrategies(CompactionStrategyManager manager, Class<? extends AbstractCompactionStrategy> expected)
{
boolean found = false;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org