You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by pa...@apache.org on 2018/06/06 18:46:51 UTC
[1/6] cassandra git commit: Re-examine commented out Compactions and
AntiCompactionsTest
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 214a3abfc -> b8fb29a6b
refs/heads/cassandra-3.11 77a12053b -> 02e9ddfad
refs/heads/trunk 843a5fdf2 -> d3b6a67bb
Re-examine commented out Compactions and AntiCompactionsTest
Patch by Lerh Chuan Low; Reviewed by Paulo Motta for CASSANDRA-13698
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b8fb29a6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b8fb29a6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b8fb29a6
Branch: refs/heads/cassandra-3.0
Commit: b8fb29a6ba1304010ac78df3cecba6ab2fece4cd
Parents: 214a3ab
Author: Lerh Chuan Low <le...@instaclustr.com>
Authored: Fri Feb 9 11:14:12 2018 -0800
Committer: Paulo Motta <pa...@gmail.com>
Committed: Wed Jun 6 15:36:59 2018 -0300
----------------------------------------------------------------------
.../org/apache/cassandra/db/SystemKeyspace.java | 2 -
.../db/compaction/AntiCompactionTest.java | 13 +-
.../db/compaction/CompactionsTest.java | 328 ++++++++-----------
3 files changed, 141 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b8fb29a6/src/java/org/apache/cassandra/db/SystemKeyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index 7ce74a1..5f5041c 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -40,8 +40,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.util.concurrent.Futures;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b8fb29a6/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
index 841a22e..ead0349 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
@@ -199,18 +199,7 @@ public class AntiCompactionTest
}
@Test
- public void antiCompactTenSTC() throws InterruptedException, IOException
- {
- antiCompactTen("SizeTieredCompactionStrategy");
- }
-
- @Test
- public void antiCompactTenLC() throws InterruptedException, IOException
- {
- antiCompactTen("LeveledCompactionStrategy");
- }
-
- public void antiCompactTen(String compactionStrategy) throws InterruptedException, IOException
+ public void antiCompactTen() throws InterruptedException, IOException
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b8fb29a6/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index 1530741..28725c7 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -18,6 +18,8 @@
*/
package org.apache.cassandra.db.compaction;
+import java.io.File;
+import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.TimeUnit;
@@ -30,13 +32,32 @@ import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.columniterator.SSTableIterator;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.dht.*;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -75,13 +96,15 @@ public class CompactionsTest
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD4),
- SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, AsciiType.instance),
- SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER5, AsciiType.instance),
- SchemaLoader.superCFMD(KEYSPACE1, CF_SUPERGC, AsciiType.instance)
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPER1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPER5),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPERGC)
.gcGraceSeconds(0));
}
- public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception
+ // Test to see if sstable has enough expired columns, it is compacted itself.
+ @Test
+ public void testSingleSSTableCompaction() throws Exception
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_DENSE1);
@@ -115,8 +138,6 @@ public class CompactionsTest
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
-
- return store;
}
public static long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl)
@@ -138,69 +159,57 @@ public class CompactionsTest
return timestamp;
}
- // Test to see if sstable has enough expired columns, it is compacted itself.
- @Test
- public void testSingleSSTableCompactionWithSizeTieredCompaction() throws Exception
- {
- testSingleSSTableCompaction(SizeTieredCompactionStrategy.class.getCanonicalName());
- }
-
- /*
- @Test
- public void testSingleSSTableCompactionWithLeveledCompaction() throws Exception
- {
- ColumnFamilyStore store = testSingleSSTableCompaction(LeveledCompactionStrategy.class.getCanonicalName());
- CompactionStrategyManager strategyManager = store.getCompactionStrategyManager();
- // tombstone removal compaction should not promote level
- assert strategyManager.getSSTableCountPerLevel()[0] == 1;
- }
-
@Test
public void testSuperColumnTombstones()
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
- CFMetaData cfm = cfs.metadata;
+ CFMetaData table = cfs.metadata;
cfs.disableAutoCompaction();
DecoratedKey key = Util.dk("tskey");
ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");
// a subcolumn
- new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), key.getKey())
- .clustering(ByteBufferUtil.bytes("cols"))
- .add("val", "val1")
- .build().applyUnsafe();
+ new RowUpdateBuilder(table, FBUtilities.timestampMicros(), key.getKey())
+ .clustering(ByteBufferUtil.bytes("cols"))
+ .add("val", "val1")
+ .build().applyUnsafe();
cfs.forceBlockingFlush();
// shadow the subcolumn with a supercolumn tombstone
- RowUpdateBuilder.deleteRow(cfm, FBUtilities.timestampMicros(), key.getKey(), ByteBufferUtil.bytes("cols")).applyUnsafe();
+ RowUpdateBuilder.deleteRow(table, FBUtilities.timestampMicros(), key.getKey(), ByteBufferUtil.bytes("cols")).applyUnsafe();
cfs.forceBlockingFlush();
- CompactionManager.instance.performMaximal(cfs);
+ CompactionManager.instance.performMaximal(cfs, false);
assertEquals(1, cfs.getLiveSSTables().size());
// check that the shadowed column is gone
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- AbstractBounds<PartitionPosition> bounds = new Bounds<PartitionPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
- ISSTableScanner scanner = sstable.getScanner(FBUtilities.nowInSeconds());
- UnfilteredRowIterator ai = scanner.next();
- assertTrue(ai.next() instanceof RangeTombstone);
- assertFalse(ai.hasNext());
-
+ AbstractBounds<PartitionPosition> bounds = new Bounds<>(key, sstable.getPartitioner().getMinimumToken().maxKeyBound());
+ UnfilteredRowIterator ai;
+ try (ISSTableScanner scanner = sstable.getScanner())
+ {
+ ai = scanner.next();
+ final Unfiltered next = ai.next();
+ assertTrue(next.isRow());
+ assertFalse(ai.hasNext());
+ }
}
@Test
public void testUncheckedTombstoneSizeTieredCompaction() throws Exception
{
+ Map<String, String> compactionOptions = new HashMap<>();
+ compactionOptions.put("tombstone_compaction_interval", "1");
+ compactionOptions.put("unchecked_tombstone_compaction", "false");
+
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
store.clearUnsafe();
- store.metadata.gcGraceSeconds(1);
- store.metadata.compactionStrategyOptions.put("tombstone_compaction_interval", "1");
- store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "false");
+
+ MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
store.reload();
- store.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getName());
// disable compaction while flushing
store.disableAutoCompaction();
@@ -237,12 +246,13 @@ public class CompactionsTest
long newSize1 = it.next().uncompressedLength();
long newSize2 = it.next().uncompressedLength();
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize1, newSize1);
+ originalSize1, newSize1);
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize2, newSize2);
+ originalSize2, newSize2);
// now let's enable the magic property
- store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "true");
+ compactionOptions.put("unchecked_tombstone_compaction", "true");
+ MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
store.reload();
//submit background task again and wait for it to complete
@@ -263,7 +273,6 @@ public class CompactionsTest
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp2);
}
- */
public static void assertMaxTimestamp(ColumnFamilyStore cfs, long maxTimestampExpected)
{
@@ -273,68 +282,13 @@ public class CompactionsTest
assertEquals(maxTimestampExpected, maxTimestampObserved);
}
- /*
@Test
- public void testEchoedRow()
+ public void testDontPurgeAccidentally() throws InterruptedException
{
- // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653
-
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
-
- // disable compaction while flushing
- cfs.disableAutoCompaction();
-
- // Insert 4 keys in two sstables. We need the sstables to have 2 rows
- // at least to trigger what was causing CASSANDRA-2653
- for (int i=1; i < 5; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
- rm.applyUnsafe();
-
- if (i % 2 == 0)
- cfs.forceBlockingFlush();
- }
- Collection<SSTableReader> toCompact = cfs.getLiveSSTables();
- assertEquals(2, toCompact.size());
-
- // Reinserting the same keys. We will compact only the previous sstable, but we need those new ones
- // to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
- for (int i=1; i < 5; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
- rm.applyUnsafe();
- }
- cfs.forceBlockingFlush();
- SSTableReader tmpSSTable = null;
- for (SSTableReader sstable : cfs.getLiveSSTables())
- if (!toCompact.contains(sstable))
- tmpSSTable = sstable;
- assertNotNull(tmpSSTable);
-
- // Force compaction on first sstables. Since each row is in only one sstable, we will be using EchoedRow.
- Util.compact(cfs, toCompact);
- assertEquals(2, cfs.getLiveSSTables().size());
-
- // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it doesn't hide the problem)
- cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
- assertEquals(1, cfs.getLiveSSTables().size());
-
- // Now assert we do have the 4 keys
- assertEquals(4, Util.getRangeSlice(cfs).size());
- }
-
- @Test
- public void testDontPurgeAccidentaly() throws InterruptedException
- {
- testDontPurgeAccidentaly("test1", "Super5");
+ testDontPurgeAccidentally("test1", CF_SUPER5);
// Use CF with gc_grace=0, see last bug of CASSANDRA-2786
- testDontPurgeAccidentaly("test1", "SuperDirectGC");
+ testDontPurgeAccidentally("test1", CF_SUPERGC);
}
@Test
@@ -343,6 +297,7 @@ public class CompactionsTest
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ CFMetaData table = cfs.metadata;
// disable compaction while flushing
cfs.disableAutoCompaction();
@@ -350,11 +305,10 @@ public class CompactionsTest
final int ROWS_PER_SSTABLE = 10;
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add(cfname, Util.cellname("col"),
- ByteBufferUtil.EMPTY_BYTE_BUFFER,
- System.currentTimeMillis());
- rm.applyUnsafe();
+ new RowUpdateBuilder(table, FBUtilities.timestampMicros(), key.getKey())
+ .clustering(ByteBufferUtil.bytes("cols"))
+ .add("val", "val1")
+ .build().applyUnsafe();
}
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getLiveSSTables();
@@ -377,6 +331,23 @@ public class CompactionsTest
assertEquals( prevGeneration + 1, sstables.iterator().next().descriptor.generation);
}
+ public static void writeSSTableWithRangeTombstoneMaskingOneColumn(ColumnFamilyStore cfs, CFMetaData table, int[] dks) {
+ for (int dk : dks)
+ {
+ RowUpdateBuilder deletedRowUpdateBuilder = new RowUpdateBuilder(table, 1, Util.dk(Integer.toString(dk)));
+ deletedRowUpdateBuilder.clustering("01").add("val", "a"); //Range tombstone covers this (timestamp 2 > 1)
+ Clustering startClustering = new Clustering(ByteBufferUtil.bytes("0"));
+ Clustering endClustering = new Clustering(ByteBufferUtil.bytes("b"));
+ deletedRowUpdateBuilder.addRangeTombstone(new RangeTombstone(Slice.make(startClustering, endClustering), new DeletionTime(2, (int) (System.currentTimeMillis() / 1000))));
+ deletedRowUpdateBuilder.build().applyUnsafe();
+
+ RowUpdateBuilder notYetDeletedRowUpdateBuilder = new RowUpdateBuilder(table, 3, Util.dk(Integer.toString(dk)));
+ notYetDeletedRowUpdateBuilder.clustering("02").add("val", "a"); //Range tombstone doesn't cover this (timestamp 3 > 2)
+ notYetDeletedRowUpdateBuilder.build().applyUnsafe();
+ }
+ cfs.forceBlockingFlush();
+ }
+
@Test
public void testRangeTombstones()
{
@@ -387,104 +358,67 @@ public class CompactionsTest
// disable compaction while flushing
cfs.disableAutoCompaction();
- final CFMetaData cfmeta = cfs.metadata;
- Directories dir = cfs.directories;
+ final CFMetaData table = cfs.metadata;
+ Directories dir = cfs.getDirectories();
ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>();
for (int i=0; i < 4; i++)
{
- keys.add(Util.dk(""+i));
+ keys.add(Util.dk(Integer.toString(i)));
}
- ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfmeta);
- cf.addColumn(Util.column("01", "a", 1)); // this must not resurrect
- cf.addColumn(Util.column("a", "a", 3));
- cf.deletionInfo().add(new RangeTombstone(Util.cellname("0"), Util.cellname("b"), 2, (int) (System.currentTimeMillis()/1000)),cfmeta.comparator);
+ int[] dks = {0, 1, 3};
+ writeSSTableWithRangeTombstoneMaskingOneColumn(cfs, table, dks);
- Descriptor desc = Descriptor.fromFilename(cfs.getSSTablePath(dir.getDirectoryForNewSSTables()));
- try(SSTableTxnWriter writer = SSTableTxnWriter.create(desc, 0, 0, 0))
- {
- writer.append(Util.dk("0"), cf);
- writer.append(Util.dk("1"), cf);
- writer.append(Util.dk("3"), cf);
-
- cfs.addSSTable(writer.closeAndOpenReader());
- }
-
- desc = Descriptor.fromFilename(cfs.getSSTablePath(dir.getDirectoryForNewSSTables()));
- try (SSTableTxnWriter writer = SSTableTxnWriter.create(desc, 0, 0, 0))
- {
- writer.append(Util.dk("0"), cf);
- writer.append(Util.dk("1"), cf);
- writer.append(Util.dk("2"), cf);
- writer.append(Util.dk("3"), cf);
- cfs.addSSTable(writer.closeAndOpenReader());
- }
+ int[] dkays = {0, 1, 2, 3};
+ writeSSTableWithRangeTombstoneMaskingOneColumn(cfs, table, dkays);
Collection<SSTableReader> toCompact = cfs.getLiveSSTables();
assert toCompact.size() == 2;
- // Force compaction on first sstables. Since each row is in only one sstable, we will be using EchoedRow.
Util.compact(cfs, toCompact);
assertEquals(1, cfs.getLiveSSTables().size());
// Now assert we do have the 4 keys
- assertEquals(4, Util.getRangeSlice(cfs).size());
+ assertEquals(4, Util.getAll(Util.cmd(cfs).build()).size());
+
+ ArrayList<DecoratedKey> k = new ArrayList<>();
- ArrayList<DecoratedKey> k = new ArrayList<DecoratedKey>();
- for (Row r : Util.getRangeSlice(cfs))
+ for (FilteredPartition p : Util.getAll(Util.cmd(cfs).build()))
{
- k.add(r.key);
- assertEquals(ByteBufferUtil.bytes("a"),r.cf.getColumn(Util.cellname("a")).value());
- assertNull(r.cf.getColumn(Util.cellname("01")));
- assertEquals(3,r.cf.getColumn(Util.cellname("a")).timestamp());
+ k.add(p.partitionKey());
+ final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
+ try (ReadOrderGroup orderGroup = command.startOrderGroup();
+ PartitionIterator iterator = command.executeInternal(orderGroup))
+ {
+ try (RowIterator rowIterator = iterator.next())
+ {
+ Row row = rowIterator.next();
+ Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("val", false)));
+ assertEquals(ByteBufferUtil.bytes("a"), cell.value());
+ assertEquals(3, cell.timestamp());
+ assertNotSame(ByteBufferUtil.bytes("01"), row.clustering().getRawValues()[0]);
+ assertEquals(ByteBufferUtil.bytes("02"), row.clustering().getRawValues()[0]);
+ }
+ }
}
-
for (SSTableReader sstable : cfs.getLiveSSTables())
{
StatsMetadata stats = sstable.getSSTableMetadata();
- assertEquals(ByteBufferUtil.bytes("0"), stats.minColumnNames.get(0));
- assertEquals(ByteBufferUtil.bytes("b"), stats.maxColumnNames.get(0));
+ assertEquals(ByteBufferUtil.bytes("0"), stats.minClusteringValues.get(0));
+ assertEquals(ByteBufferUtil.bytes("b"), stats.maxClusteringValues.get(0));
}
assertEquals(keys, k);
}
- @Test
- public void testCompactionLog() throws Exception
- {
- SystemKeyspace.discardCompactionsInProgress();
-
- String cf = "Standard4";
- ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
- SchemaLoader.insertData(KEYSPACE1, cf, 0, 1);
- cfs.forceBlockingFlush();
-
- Collection<SSTableReader> sstables = cfs.getLiveSSTables();
- assertFalse(sstables.isEmpty());
- Set<Integer> generations = Sets.newHashSet(Iterables.transform(sstables, new Function<SSTableReader, Integer>()
- {
- public Integer apply(SSTableReader sstable)
- {
- return sstable.descriptor.generation;
- }
- }));
- UUID taskId = SystemKeyspace.startCompaction(cfs, sstables);
- Map<Pair<String, String>, Map<Integer, UUID>> compactionLogs = SystemKeyspace.getUnfinishedCompactions();
- Set<Integer> unfinishedCompactions = compactionLogs.get(Pair.create(KEYSPACE1, cf)).keySet();
- assertTrue(unfinishedCompactions.containsAll(generations));
-
- SystemKeyspace.finishCompaction(taskId);
- compactionLogs = SystemKeyspace.getUnfinishedCompactions();
- assertFalse(compactionLogs.containsKey(Pair.create(KEYSPACE1, cf)));
- }
-
- private void testDontPurgeAccidentaly(String k, String cfname) throws InterruptedException
+ private void testDontPurgeAccidentally(String k, String cfname) throws InterruptedException
{
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ CFMetaData table = cfs.metadata;
// disable compaction while flushing
cfs.clearUnsafe();
@@ -492,24 +426,24 @@ public class CompactionsTest
// Add test row
DecoratedKey key = Util.dk(k);
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add(cfname, Util.cellname(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0);
- rm.applyUnsafe();
+ RowUpdateBuilder rowUpdateBuilder = new RowUpdateBuilder(table, 0, key);
+ rowUpdateBuilder.clustering("c").add("val", "a");
+ rowUpdateBuilder.build().applyUnsafe();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesBefore = cfs.getLiveSSTables();
- QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
- assertTrue(cfs.getColumnFamily(filter).hasColumns());
+ ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+ assertTrue(!partition.isEmpty());
+ RowUpdateBuilder deleteRowBuilder = new RowUpdateBuilder(table, 2, key);
+ deleteRowBuilder.clustering("c").delete("val");
+ deleteRowBuilder.build().applyUnsafe();
// Remove key
- rm = new Mutation(KEYSPACE1, key.getKey());
- rm.delete(cfname, 2);
- rm.applyUnsafe();
- ColumnFamily cf = cfs.getColumnFamily(filter);
- assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns());
+ partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+ assertTrue(partition.iterator().next().cells().iterator().next().isTombstone());
// Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
Thread.sleep(1000);
@@ -524,10 +458,28 @@ public class CompactionsTest
Util.compact(cfs, toCompact);
- cf = cfs.getColumnFamily(filter);
- assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns());
+ SSTableReader newSSTable = null;
+ for (SSTableReader reader : cfs.getLiveSSTables())
+ {
+ assert !toCompact.contains(reader);
+ if (!sstablesBefore.contains(reader))
+ newSSTable = reader;
+ }
+
+ // We cannot read the data, since {@link ReadCommand#withoutPurgeableTombstones} will purge droppable tombstones
+ // but we just want to check here that compaction did *NOT* drop the tombstone, so we read from the SSTable directly
+ // instead
+ ISSTableScanner scanner = newSSTable.getScanner();
+ assertTrue(scanner.hasNext());
+ UnfilteredRowIterator rowIt = scanner.next();
+ assertTrue(rowIt.hasNext());
+ Unfiltered unfiltered = rowIt.next();
+ assertTrue(unfiltered.isRow());
+ Row row = (Row)unfiltered;
+ assertTrue(row.cells().iterator().next().isTombstone());
+ assertFalse(rowIt.hasNext());
+ assertFalse(scanner.hasNext());
}
- */
private static Range<Token> rangeFor(int start, int end)
{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[3/6] cassandra git commit: Re-examine commented out Compactions and
AntiCompactionsTest
Posted by pa...@apache.org.
Re-examine commented out Compactions and AntiCompactionsTest
Patch by Lerh Chuan Low; Reviewed by Paulo Motta for CASSANDRA-13698
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b8fb29a6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b8fb29a6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b8fb29a6
Branch: refs/heads/trunk
Commit: b8fb29a6ba1304010ac78df3cecba6ab2fece4cd
Parents: 214a3ab
Author: Lerh Chuan Low <le...@instaclustr.com>
Authored: Fri Feb 9 11:14:12 2018 -0800
Committer: Paulo Motta <pa...@gmail.com>
Committed: Wed Jun 6 15:36:59 2018 -0300
----------------------------------------------------------------------
.../org/apache/cassandra/db/SystemKeyspace.java | 2 -
.../db/compaction/AntiCompactionTest.java | 13 +-
.../db/compaction/CompactionsTest.java | 328 ++++++++-----------
3 files changed, 141 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b8fb29a6/src/java/org/apache/cassandra/db/SystemKeyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index 7ce74a1..5f5041c 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -40,8 +40,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.util.concurrent.Futures;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b8fb29a6/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
index 841a22e..ead0349 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
@@ -199,18 +199,7 @@ public class AntiCompactionTest
}
@Test
- public void antiCompactTenSTC() throws InterruptedException, IOException
- {
- antiCompactTen("SizeTieredCompactionStrategy");
- }
-
- @Test
- public void antiCompactTenLC() throws InterruptedException, IOException
- {
- antiCompactTen("LeveledCompactionStrategy");
- }
-
- public void antiCompactTen(String compactionStrategy) throws InterruptedException, IOException
+ public void antiCompactTen() throws InterruptedException, IOException
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b8fb29a6/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index 1530741..28725c7 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -18,6 +18,8 @@
*/
package org.apache.cassandra.db.compaction;
+import java.io.File;
+import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.TimeUnit;
@@ -30,13 +32,32 @@ import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.columniterator.SSTableIterator;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.dht.*;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -75,13 +96,15 @@ public class CompactionsTest
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD4),
- SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, AsciiType.instance),
- SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER5, AsciiType.instance),
- SchemaLoader.superCFMD(KEYSPACE1, CF_SUPERGC, AsciiType.instance)
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPER1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPER5),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPERGC)
.gcGraceSeconds(0));
}
- public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception
+ // Test to see if sstable has enough expired columns, it is compacted itself.
+ @Test
+ public void testSingleSSTableCompaction() throws Exception
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_DENSE1);
@@ -115,8 +138,6 @@ public class CompactionsTest
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
-
- return store;
}
public static long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl)
@@ -138,69 +159,57 @@ public class CompactionsTest
return timestamp;
}
- // Test to see if sstable has enough expired columns, it is compacted itself.
- @Test
- public void testSingleSSTableCompactionWithSizeTieredCompaction() throws Exception
- {
- testSingleSSTableCompaction(SizeTieredCompactionStrategy.class.getCanonicalName());
- }
-
- /*
- @Test
- public void testSingleSSTableCompactionWithLeveledCompaction() throws Exception
- {
- ColumnFamilyStore store = testSingleSSTableCompaction(LeveledCompactionStrategy.class.getCanonicalName());
- CompactionStrategyManager strategyManager = store.getCompactionStrategyManager();
- // tombstone removal compaction should not promote level
- assert strategyManager.getSSTableCountPerLevel()[0] == 1;
- }
-
@Test
public void testSuperColumnTombstones()
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
- CFMetaData cfm = cfs.metadata;
+ CFMetaData table = cfs.metadata;
cfs.disableAutoCompaction();
DecoratedKey key = Util.dk("tskey");
ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");
// a subcolumn
- new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), key.getKey())
- .clustering(ByteBufferUtil.bytes("cols"))
- .add("val", "val1")
- .build().applyUnsafe();
+ new RowUpdateBuilder(table, FBUtilities.timestampMicros(), key.getKey())
+ .clustering(ByteBufferUtil.bytes("cols"))
+ .add("val", "val1")
+ .build().applyUnsafe();
cfs.forceBlockingFlush();
// shadow the subcolumn with a supercolumn tombstone
- RowUpdateBuilder.deleteRow(cfm, FBUtilities.timestampMicros(), key.getKey(), ByteBufferUtil.bytes("cols")).applyUnsafe();
+ RowUpdateBuilder.deleteRow(table, FBUtilities.timestampMicros(), key.getKey(), ByteBufferUtil.bytes("cols")).applyUnsafe();
cfs.forceBlockingFlush();
- CompactionManager.instance.performMaximal(cfs);
+ CompactionManager.instance.performMaximal(cfs, false);
assertEquals(1, cfs.getLiveSSTables().size());
// check that the shadowed column is gone
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- AbstractBounds<PartitionPosition> bounds = new Bounds<PartitionPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
- ISSTableScanner scanner = sstable.getScanner(FBUtilities.nowInSeconds());
- UnfilteredRowIterator ai = scanner.next();
- assertTrue(ai.next() instanceof RangeTombstone);
- assertFalse(ai.hasNext());
-
+ AbstractBounds<PartitionPosition> bounds = new Bounds<>(key, sstable.getPartitioner().getMinimumToken().maxKeyBound());
+ UnfilteredRowIterator ai;
+ try (ISSTableScanner scanner = sstable.getScanner())
+ {
+ ai = scanner.next();
+ final Unfiltered next = ai.next();
+ assertTrue(next.isRow());
+ assertFalse(ai.hasNext());
+ }
}
@Test
public void testUncheckedTombstoneSizeTieredCompaction() throws Exception
{
+ Map<String, String> compactionOptions = new HashMap<>();
+ compactionOptions.put("tombstone_compaction_interval", "1");
+ compactionOptions.put("unchecked_tombstone_compaction", "false");
+
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
store.clearUnsafe();
- store.metadata.gcGraceSeconds(1);
- store.metadata.compactionStrategyOptions.put("tombstone_compaction_interval", "1");
- store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "false");
+
+ MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
store.reload();
- store.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getName());
// disable compaction while flushing
store.disableAutoCompaction();
@@ -237,12 +246,13 @@ public class CompactionsTest
long newSize1 = it.next().uncompressedLength();
long newSize2 = it.next().uncompressedLength();
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize1, newSize1);
+ originalSize1, newSize1);
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize2, newSize2);
+ originalSize2, newSize2);
// now let's enable the magic property
- store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "true");
+ compactionOptions.put("unchecked_tombstone_compaction", "true");
+ MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
store.reload();
//submit background task again and wait for it to complete
@@ -263,7 +273,6 @@ public class CompactionsTest
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp2);
}
- */
public static void assertMaxTimestamp(ColumnFamilyStore cfs, long maxTimestampExpected)
{
@@ -273,68 +282,13 @@ public class CompactionsTest
assertEquals(maxTimestampExpected, maxTimestampObserved);
}
- /*
@Test
- public void testEchoedRow()
+ public void testDontPurgeAccidentally() throws InterruptedException
{
- // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653
-
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
-
- // disable compaction while flushing
- cfs.disableAutoCompaction();
-
- // Insert 4 keys in two sstables. We need the sstables to have 2 rows
- // at least to trigger what was causing CASSANDRA-2653
- for (int i=1; i < 5; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
- rm.applyUnsafe();
-
- if (i % 2 == 0)
- cfs.forceBlockingFlush();
- }
- Collection<SSTableReader> toCompact = cfs.getLiveSSTables();
- assertEquals(2, toCompact.size());
-
- // Reinserting the same keys. We will compact only the previous sstable, but we need those new ones
- // to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
- for (int i=1; i < 5; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
- rm.applyUnsafe();
- }
- cfs.forceBlockingFlush();
- SSTableReader tmpSSTable = null;
- for (SSTableReader sstable : cfs.getLiveSSTables())
- if (!toCompact.contains(sstable))
- tmpSSTable = sstable;
- assertNotNull(tmpSSTable);
-
- // Force compaction on first sstables. Since each row is in only one sstable, we will be using EchoedRow.
- Util.compact(cfs, toCompact);
- assertEquals(2, cfs.getLiveSSTables().size());
-
- // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it doesn't hide the problem)
- cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
- assertEquals(1, cfs.getLiveSSTables().size());
-
- // Now assert we do have the 4 keys
- assertEquals(4, Util.getRangeSlice(cfs).size());
- }
-
- @Test
- public void testDontPurgeAccidentaly() throws InterruptedException
- {
- testDontPurgeAccidentaly("test1", "Super5");
+ testDontPurgeAccidentally("test1", CF_SUPER5);
// Use CF with gc_grace=0, see last bug of CASSANDRA-2786
- testDontPurgeAccidentaly("test1", "SuperDirectGC");
+ testDontPurgeAccidentally("test1", CF_SUPERGC);
}
@Test
@@ -343,6 +297,7 @@ public class CompactionsTest
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ CFMetaData table = cfs.metadata;
// disable compaction while flushing
cfs.disableAutoCompaction();
@@ -350,11 +305,10 @@ public class CompactionsTest
final int ROWS_PER_SSTABLE = 10;
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add(cfname, Util.cellname("col"),
- ByteBufferUtil.EMPTY_BYTE_BUFFER,
- System.currentTimeMillis());
- rm.applyUnsafe();
+ new RowUpdateBuilder(table, FBUtilities.timestampMicros(), key.getKey())
+ .clustering(ByteBufferUtil.bytes("cols"))
+ .add("val", "val1")
+ .build().applyUnsafe();
}
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getLiveSSTables();
@@ -377,6 +331,23 @@ public class CompactionsTest
assertEquals( prevGeneration + 1, sstables.iterator().next().descriptor.generation);
}
+ public static void writeSSTableWithRangeTombstoneMaskingOneColumn(ColumnFamilyStore cfs, CFMetaData table, int[] dks) {
+ for (int dk : dks)
+ {
+ RowUpdateBuilder deletedRowUpdateBuilder = new RowUpdateBuilder(table, 1, Util.dk(Integer.toString(dk)));
+ deletedRowUpdateBuilder.clustering("01").add("val", "a"); //Range tombstone covers this (timestamp 2 > 1)
+ Clustering startClustering = new Clustering(ByteBufferUtil.bytes("0"));
+ Clustering endClustering = new Clustering(ByteBufferUtil.bytes("b"));
+ deletedRowUpdateBuilder.addRangeTombstone(new RangeTombstone(Slice.make(startClustering, endClustering), new DeletionTime(2, (int) (System.currentTimeMillis() / 1000))));
+ deletedRowUpdateBuilder.build().applyUnsafe();
+
+ RowUpdateBuilder notYetDeletedRowUpdateBuilder = new RowUpdateBuilder(table, 3, Util.dk(Integer.toString(dk)));
+ notYetDeletedRowUpdateBuilder.clustering("02").add("val", "a"); //Range tombstone doesn't cover this (timestamp 3 > 2)
+ notYetDeletedRowUpdateBuilder.build().applyUnsafe();
+ }
+ cfs.forceBlockingFlush();
+ }
+
@Test
public void testRangeTombstones()
{
@@ -387,104 +358,67 @@ public class CompactionsTest
// disable compaction while flushing
cfs.disableAutoCompaction();
- final CFMetaData cfmeta = cfs.metadata;
- Directories dir = cfs.directories;
+ final CFMetaData table = cfs.metadata;
+ Directories dir = cfs.getDirectories();
ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>();
for (int i=0; i < 4; i++)
{
- keys.add(Util.dk(""+i));
+ keys.add(Util.dk(Integer.toString(i)));
}
- ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfmeta);
- cf.addColumn(Util.column("01", "a", 1)); // this must not resurrect
- cf.addColumn(Util.column("a", "a", 3));
- cf.deletionInfo().add(new RangeTombstone(Util.cellname("0"), Util.cellname("b"), 2, (int) (System.currentTimeMillis()/1000)),cfmeta.comparator);
+ int[] dks = {0, 1, 3};
+ writeSSTableWithRangeTombstoneMaskingOneColumn(cfs, table, dks);
- Descriptor desc = Descriptor.fromFilename(cfs.getSSTablePath(dir.getDirectoryForNewSSTables()));
- try(SSTableTxnWriter writer = SSTableTxnWriter.create(desc, 0, 0, 0))
- {
- writer.append(Util.dk("0"), cf);
- writer.append(Util.dk("1"), cf);
- writer.append(Util.dk("3"), cf);
-
- cfs.addSSTable(writer.closeAndOpenReader());
- }
-
- desc = Descriptor.fromFilename(cfs.getSSTablePath(dir.getDirectoryForNewSSTables()));
- try (SSTableTxnWriter writer = SSTableTxnWriter.create(desc, 0, 0, 0))
- {
- writer.append(Util.dk("0"), cf);
- writer.append(Util.dk("1"), cf);
- writer.append(Util.dk("2"), cf);
- writer.append(Util.dk("3"), cf);
- cfs.addSSTable(writer.closeAndOpenReader());
- }
+ int[] dkays = {0, 1, 2, 3};
+ writeSSTableWithRangeTombstoneMaskingOneColumn(cfs, table, dkays);
Collection<SSTableReader> toCompact = cfs.getLiveSSTables();
assert toCompact.size() == 2;
- // Force compaction on first sstables. Since each row is in only one sstable, we will be using EchoedRow.
Util.compact(cfs, toCompact);
assertEquals(1, cfs.getLiveSSTables().size());
// Now assert we do have the 4 keys
- assertEquals(4, Util.getRangeSlice(cfs).size());
+ assertEquals(4, Util.getAll(Util.cmd(cfs).build()).size());
+
+ ArrayList<DecoratedKey> k = new ArrayList<>();
- ArrayList<DecoratedKey> k = new ArrayList<DecoratedKey>();
- for (Row r : Util.getRangeSlice(cfs))
+ for (FilteredPartition p : Util.getAll(Util.cmd(cfs).build()))
{
- k.add(r.key);
- assertEquals(ByteBufferUtil.bytes("a"),r.cf.getColumn(Util.cellname("a")).value());
- assertNull(r.cf.getColumn(Util.cellname("01")));
- assertEquals(3,r.cf.getColumn(Util.cellname("a")).timestamp());
+ k.add(p.partitionKey());
+ final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
+ try (ReadOrderGroup orderGroup = command.startOrderGroup();
+ PartitionIterator iterator = command.executeInternal(orderGroup))
+ {
+ try (RowIterator rowIterator = iterator.next())
+ {
+ Row row = rowIterator.next();
+ Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("val", false)));
+ assertEquals(ByteBufferUtil.bytes("a"), cell.value());
+ assertEquals(3, cell.timestamp());
+ assertNotSame(ByteBufferUtil.bytes("01"), row.clustering().getRawValues()[0]);
+ assertEquals(ByteBufferUtil.bytes("02"), row.clustering().getRawValues()[0]);
+ }
+ }
}
-
for (SSTableReader sstable : cfs.getLiveSSTables())
{
StatsMetadata stats = sstable.getSSTableMetadata();
- assertEquals(ByteBufferUtil.bytes("0"), stats.minColumnNames.get(0));
- assertEquals(ByteBufferUtil.bytes("b"), stats.maxColumnNames.get(0));
+ assertEquals(ByteBufferUtil.bytes("0"), stats.minClusteringValues.get(0));
+ assertEquals(ByteBufferUtil.bytes("b"), stats.maxClusteringValues.get(0));
}
assertEquals(keys, k);
}
- @Test
- public void testCompactionLog() throws Exception
- {
- SystemKeyspace.discardCompactionsInProgress();
-
- String cf = "Standard4";
- ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
- SchemaLoader.insertData(KEYSPACE1, cf, 0, 1);
- cfs.forceBlockingFlush();
-
- Collection<SSTableReader> sstables = cfs.getLiveSSTables();
- assertFalse(sstables.isEmpty());
- Set<Integer> generations = Sets.newHashSet(Iterables.transform(sstables, new Function<SSTableReader, Integer>()
- {
- public Integer apply(SSTableReader sstable)
- {
- return sstable.descriptor.generation;
- }
- }));
- UUID taskId = SystemKeyspace.startCompaction(cfs, sstables);
- Map<Pair<String, String>, Map<Integer, UUID>> compactionLogs = SystemKeyspace.getUnfinishedCompactions();
- Set<Integer> unfinishedCompactions = compactionLogs.get(Pair.create(KEYSPACE1, cf)).keySet();
- assertTrue(unfinishedCompactions.containsAll(generations));
-
- SystemKeyspace.finishCompaction(taskId);
- compactionLogs = SystemKeyspace.getUnfinishedCompactions();
- assertFalse(compactionLogs.containsKey(Pair.create(KEYSPACE1, cf)));
- }
-
- private void testDontPurgeAccidentaly(String k, String cfname) throws InterruptedException
+ private void testDontPurgeAccidentally(String k, String cfname) throws InterruptedException
{
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ CFMetaData table = cfs.metadata;
// disable compaction while flushing
cfs.clearUnsafe();
@@ -492,24 +426,24 @@ public class CompactionsTest
// Add test row
DecoratedKey key = Util.dk(k);
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add(cfname, Util.cellname(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0);
- rm.applyUnsafe();
+ RowUpdateBuilder rowUpdateBuilder = new RowUpdateBuilder(table, 0, key);
+ rowUpdateBuilder.clustering("c").add("val", "a");
+ rowUpdateBuilder.build().applyUnsafe();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesBefore = cfs.getLiveSSTables();
- QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
- assertTrue(cfs.getColumnFamily(filter).hasColumns());
+ ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+ assertTrue(!partition.isEmpty());
+ RowUpdateBuilder deleteRowBuilder = new RowUpdateBuilder(table, 2, key);
+ deleteRowBuilder.clustering("c").delete("val");
+ deleteRowBuilder.build().applyUnsafe();
// Remove key
- rm = new Mutation(KEYSPACE1, key.getKey());
- rm.delete(cfname, 2);
- rm.applyUnsafe();
- ColumnFamily cf = cfs.getColumnFamily(filter);
- assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns());
+ partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+ assertTrue(partition.iterator().next().cells().iterator().next().isTombstone());
// Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
Thread.sleep(1000);
@@ -524,10 +458,28 @@ public class CompactionsTest
Util.compact(cfs, toCompact);
- cf = cfs.getColumnFamily(filter);
- assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns());
+ SSTableReader newSSTable = null;
+ for (SSTableReader reader : cfs.getLiveSSTables())
+ {
+ assert !toCompact.contains(reader);
+ if (!sstablesBefore.contains(reader))
+ newSSTable = reader;
+ }
+
+ // We cannot read the data, since {@link ReadCommand#withoutPurgeableTombstones} will purge droppable tombstones
+ // but we just want to check here that compaction did *NOT* drop the tombstone, so we read from the SSTable directly
+ // instead
+ ISSTableScanner scanner = newSSTable.getScanner();
+ assertTrue(scanner.hasNext());
+ UnfilteredRowIterator rowIt = scanner.next();
+ assertTrue(rowIt.hasNext());
+ Unfiltered unfiltered = rowIt.next();
+ assertTrue(unfiltered.isRow());
+ Row row = (Row)unfiltered;
+ assertTrue(row.cells().iterator().next().isTombstone());
+ assertFalse(rowIt.hasNext());
+ assertFalse(scanner.hasNext());
}
- */
private static Range<Token> rangeFor(int start, int end)
{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[5/6] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by pa...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/02e9ddfa
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/02e9ddfa
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/02e9ddfa
Branch: refs/heads/cassandra-3.11
Commit: 02e9ddfad9babf98d743fedd005e3ca41df8c2eb
Parents: 77a1205 b8fb29a
Author: Paulo Motta <pa...@gmail.com>
Authored: Wed Jun 6 15:37:24 2018 -0300
Committer: Paulo Motta <pa...@gmail.com>
Committed: Wed Jun 6 15:37:59 2018 -0300
----------------------------------------------------------------------
.../org/apache/cassandra/db/SystemKeyspace.java | 2 -
.../db/compaction/AntiCompactionTest.java | 13 +-
.../db/compaction/CompactionsTest.java | 328 ++++++++-----------
3 files changed, 141 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/02e9ddfa/src/java/org/apache/cassandra/db/SystemKeyspace.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/SystemKeyspace.java
index ac51662,5f5041c..973538f
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@@ -42,11 -40,8 +42,9 @@@ import org.slf4j.LoggerFactory
import com.google.common.util.concurrent.Futures;
- import org.apache.cassandra.concurrent.Stage;
- import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.functions.*;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/02e9ddfa/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[4/6] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by pa...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/02e9ddfa
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/02e9ddfa
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/02e9ddfa
Branch: refs/heads/trunk
Commit: 02e9ddfad9babf98d743fedd005e3ca41df8c2eb
Parents: 77a1205 b8fb29a
Author: Paulo Motta <pa...@gmail.com>
Authored: Wed Jun 6 15:37:24 2018 -0300
Committer: Paulo Motta <pa...@gmail.com>
Committed: Wed Jun 6 15:37:59 2018 -0300
----------------------------------------------------------------------
.../org/apache/cassandra/db/SystemKeyspace.java | 2 -
.../db/compaction/AntiCompactionTest.java | 13 +-
.../db/compaction/CompactionsTest.java | 328 ++++++++-----------
3 files changed, 141 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/02e9ddfa/src/java/org/apache/cassandra/db/SystemKeyspace.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/SystemKeyspace.java
index ac51662,5f5041c..973538f
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@@ -42,11 -40,8 +42,9 @@@ import org.slf4j.LoggerFactory
import com.google.common.util.concurrent.Futures;
- import org.apache.cassandra.concurrent.Stage;
- import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.functions.*;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/02e9ddfa/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[2/6] cassandra git commit: Re-examine commented out Compactions and
AntiCompactionsTest
Posted by pa...@apache.org.
Re-examine commented out Compactions and AntiCompactionsTest
Patch by Lerh Chuan Low; Reviewed by Paulo Motta for CASSANDRA-13698
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b8fb29a6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b8fb29a6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b8fb29a6
Branch: refs/heads/cassandra-3.11
Commit: b8fb29a6ba1304010ac78df3cecba6ab2fece4cd
Parents: 214a3ab
Author: Lerh Chuan Low <le...@instaclustr.com>
Authored: Fri Feb 9 11:14:12 2018 -0800
Committer: Paulo Motta <pa...@gmail.com>
Committed: Wed Jun 6 15:36:59 2018 -0300
----------------------------------------------------------------------
.../org/apache/cassandra/db/SystemKeyspace.java | 2 -
.../db/compaction/AntiCompactionTest.java | 13 +-
.../db/compaction/CompactionsTest.java | 328 ++++++++-----------
3 files changed, 141 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b8fb29a6/src/java/org/apache/cassandra/db/SystemKeyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index 7ce74a1..5f5041c 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -40,8 +40,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.util.concurrent.Futures;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b8fb29a6/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
index 841a22e..ead0349 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
@@ -199,18 +199,7 @@ public class AntiCompactionTest
}
@Test
- public void antiCompactTenSTC() throws InterruptedException, IOException
- {
- antiCompactTen("SizeTieredCompactionStrategy");
- }
-
- @Test
- public void antiCompactTenLC() throws InterruptedException, IOException
- {
- antiCompactTen("LeveledCompactionStrategy");
- }
-
- public void antiCompactTen(String compactionStrategy) throws InterruptedException, IOException
+ public void antiCompactTen() throws InterruptedException, IOException
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b8fb29a6/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index 1530741..28725c7 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -18,6 +18,8 @@
*/
package org.apache.cassandra.db.compaction;
+import java.io.File;
+import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.TimeUnit;
@@ -30,13 +32,32 @@ import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.columniterator.SSTableIterator;
+import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.FilteredPartition;
+import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.dht.*;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -75,13 +96,15 @@ public class CompactionsTest
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD4),
- SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER1, AsciiType.instance),
- SchemaLoader.superCFMD(KEYSPACE1, CF_SUPER5, AsciiType.instance),
- SchemaLoader.superCFMD(KEYSPACE1, CF_SUPERGC, AsciiType.instance)
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPER1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPER5),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_SUPERGC)
.gcGraceSeconds(0));
}
- public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception
+ // Test to see if sstable has enough expired columns, it is compacted itself.
+ @Test
+ public void testSingleSSTableCompaction() throws Exception
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_DENSE1);
@@ -115,8 +138,6 @@ public class CompactionsTest
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
-
- return store;
}
public static long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl)
@@ -138,69 +159,57 @@ public class CompactionsTest
return timestamp;
}
- // Test to see if sstable has enough expired columns, it is compacted itself.
- @Test
- public void testSingleSSTableCompactionWithSizeTieredCompaction() throws Exception
- {
- testSingleSSTableCompaction(SizeTieredCompactionStrategy.class.getCanonicalName());
- }
-
- /*
- @Test
- public void testSingleSSTableCompactionWithLeveledCompaction() throws Exception
- {
- ColumnFamilyStore store = testSingleSSTableCompaction(LeveledCompactionStrategy.class.getCanonicalName());
- CompactionStrategyManager strategyManager = store.getCompactionStrategyManager();
- // tombstone removal compaction should not promote level
- assert strategyManager.getSSTableCountPerLevel()[0] == 1;
- }
-
@Test
public void testSuperColumnTombstones()
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
- CFMetaData cfm = cfs.metadata;
+ CFMetaData table = cfs.metadata;
cfs.disableAutoCompaction();
DecoratedKey key = Util.dk("tskey");
ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");
// a subcolumn
- new RowUpdateBuilder(cfm, FBUtilities.timestampMicros(), key.getKey())
- .clustering(ByteBufferUtil.bytes("cols"))
- .add("val", "val1")
- .build().applyUnsafe();
+ new RowUpdateBuilder(table, FBUtilities.timestampMicros(), key.getKey())
+ .clustering(ByteBufferUtil.bytes("cols"))
+ .add("val", "val1")
+ .build().applyUnsafe();
cfs.forceBlockingFlush();
// shadow the subcolumn with a supercolumn tombstone
- RowUpdateBuilder.deleteRow(cfm, FBUtilities.timestampMicros(), key.getKey(), ByteBufferUtil.bytes("cols")).applyUnsafe();
+ RowUpdateBuilder.deleteRow(table, FBUtilities.timestampMicros(), key.getKey(), ByteBufferUtil.bytes("cols")).applyUnsafe();
cfs.forceBlockingFlush();
- CompactionManager.instance.performMaximal(cfs);
+ CompactionManager.instance.performMaximal(cfs, false);
assertEquals(1, cfs.getLiveSSTables().size());
// check that the shadowed column is gone
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- AbstractBounds<PartitionPosition> bounds = new Bounds<PartitionPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
- ISSTableScanner scanner = sstable.getScanner(FBUtilities.nowInSeconds());
- UnfilteredRowIterator ai = scanner.next();
- assertTrue(ai.next() instanceof RangeTombstone);
- assertFalse(ai.hasNext());
-
+ AbstractBounds<PartitionPosition> bounds = new Bounds<>(key, sstable.getPartitioner().getMinimumToken().maxKeyBound());
+ UnfilteredRowIterator ai;
+ try (ISSTableScanner scanner = sstable.getScanner())
+ {
+ ai = scanner.next();
+ final Unfiltered next = ai.next();
+ assertTrue(next.isRow());
+ assertFalse(ai.hasNext());
+ }
}
@Test
public void testUncheckedTombstoneSizeTieredCompaction() throws Exception
{
+ Map<String, String> compactionOptions = new HashMap<>();
+ compactionOptions.put("tombstone_compaction_interval", "1");
+ compactionOptions.put("unchecked_tombstone_compaction", "false");
+
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
store.clearUnsafe();
- store.metadata.gcGraceSeconds(1);
- store.metadata.compactionStrategyOptions.put("tombstone_compaction_interval", "1");
- store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "false");
+
+ MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
store.reload();
- store.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getName());
// disable compaction while flushing
store.disableAutoCompaction();
@@ -237,12 +246,13 @@ public class CompactionsTest
long newSize1 = it.next().uncompressedLength();
long newSize2 = it.next().uncompressedLength();
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize1, newSize1);
+ originalSize1, newSize1);
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize2, newSize2);
+ originalSize2, newSize2);
// now let's enable the magic property
- store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "true");
+ compactionOptions.put("unchecked_tombstone_compaction", "true");
+ MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
store.reload();
//submit background task again and wait for it to complete
@@ -263,7 +273,6 @@ public class CompactionsTest
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp2);
}
- */
public static void assertMaxTimestamp(ColumnFamilyStore cfs, long maxTimestampExpected)
{
@@ -273,68 +282,13 @@ public class CompactionsTest
assertEquals(maxTimestampExpected, maxTimestampObserved);
}
- /*
@Test
- public void testEchoedRow()
+ public void testDontPurgeAccidentally() throws InterruptedException
{
- // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653
-
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
-
- // disable compaction while flushing
- cfs.disableAutoCompaction();
-
- // Insert 4 keys in two sstables. We need the sstables to have 2 rows
- // at least to trigger what was causing CASSANDRA-2653
- for (int i=1; i < 5; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
- rm.applyUnsafe();
-
- if (i % 2 == 0)
- cfs.forceBlockingFlush();
- }
- Collection<SSTableReader> toCompact = cfs.getLiveSSTables();
- assertEquals(2, toCompact.size());
-
- // Reinserting the same keys. We will compact only the previous sstable, but we need those new ones
- // to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
- for (int i=1; i < 5; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
- rm.applyUnsafe();
- }
- cfs.forceBlockingFlush();
- SSTableReader tmpSSTable = null;
- for (SSTableReader sstable : cfs.getLiveSSTables())
- if (!toCompact.contains(sstable))
- tmpSSTable = sstable;
- assertNotNull(tmpSSTable);
-
- // Force compaction on first sstables. Since each row is in only one sstable, we will be using EchoedRow.
- Util.compact(cfs, toCompact);
- assertEquals(2, cfs.getLiveSSTables().size());
-
- // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it doesn't hide the problem)
- cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
- assertEquals(1, cfs.getLiveSSTables().size());
-
- // Now assert we do have the 4 keys
- assertEquals(4, Util.getRangeSlice(cfs).size());
- }
-
- @Test
- public void testDontPurgeAccidentaly() throws InterruptedException
- {
- testDontPurgeAccidentaly("test1", "Super5");
+ testDontPurgeAccidentally("test1", CF_SUPER5);
// Use CF with gc_grace=0, see last bug of CASSANDRA-2786
- testDontPurgeAccidentaly("test1", "SuperDirectGC");
+ testDontPurgeAccidentally("test1", CF_SUPERGC);
}
@Test
@@ -343,6 +297,7 @@ public class CompactionsTest
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ CFMetaData table = cfs.metadata;
// disable compaction while flushing
cfs.disableAutoCompaction();
@@ -350,11 +305,10 @@ public class CompactionsTest
final int ROWS_PER_SSTABLE = 10;
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add(cfname, Util.cellname("col"),
- ByteBufferUtil.EMPTY_BYTE_BUFFER,
- System.currentTimeMillis());
- rm.applyUnsafe();
+ new RowUpdateBuilder(table, FBUtilities.timestampMicros(), key.getKey())
+ .clustering(ByteBufferUtil.bytes("cols"))
+ .add("val", "val1")
+ .build().applyUnsafe();
}
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getLiveSSTables();
@@ -377,6 +331,23 @@ public class CompactionsTest
assertEquals( prevGeneration + 1, sstables.iterator().next().descriptor.generation);
}
+ public static void writeSSTableWithRangeTombstoneMaskingOneColumn(ColumnFamilyStore cfs, CFMetaData table, int[] dks) {
+ for (int dk : dks)
+ {
+ RowUpdateBuilder deletedRowUpdateBuilder = new RowUpdateBuilder(table, 1, Util.dk(Integer.toString(dk)));
+ deletedRowUpdateBuilder.clustering("01").add("val", "a"); //Range tombstone covers this (timestamp 2 > 1)
+ Clustering startClustering = new Clustering(ByteBufferUtil.bytes("0"));
+ Clustering endClustering = new Clustering(ByteBufferUtil.bytes("b"));
+ deletedRowUpdateBuilder.addRangeTombstone(new RangeTombstone(Slice.make(startClustering, endClustering), new DeletionTime(2, (int) (System.currentTimeMillis() / 1000))));
+ deletedRowUpdateBuilder.build().applyUnsafe();
+
+ RowUpdateBuilder notYetDeletedRowUpdateBuilder = new RowUpdateBuilder(table, 3, Util.dk(Integer.toString(dk)));
+ notYetDeletedRowUpdateBuilder.clustering("02").add("val", "a"); //Range tombstone doesn't cover this (timestamp 3 > 2)
+ notYetDeletedRowUpdateBuilder.build().applyUnsafe();
+ }
+ cfs.forceBlockingFlush();
+ }
+
@Test
public void testRangeTombstones()
{
@@ -387,104 +358,67 @@ public class CompactionsTest
// disable compaction while flushing
cfs.disableAutoCompaction();
- final CFMetaData cfmeta = cfs.metadata;
- Directories dir = cfs.directories;
+ final CFMetaData table = cfs.metadata;
+ Directories dir = cfs.getDirectories();
ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>();
for (int i=0; i < 4; i++)
{
- keys.add(Util.dk(""+i));
+ keys.add(Util.dk(Integer.toString(i)));
}
- ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfmeta);
- cf.addColumn(Util.column("01", "a", 1)); // this must not resurrect
- cf.addColumn(Util.column("a", "a", 3));
- cf.deletionInfo().add(new RangeTombstone(Util.cellname("0"), Util.cellname("b"), 2, (int) (System.currentTimeMillis()/1000)),cfmeta.comparator);
+ int[] dks = {0, 1, 3};
+ writeSSTableWithRangeTombstoneMaskingOneColumn(cfs, table, dks);
- Descriptor desc = Descriptor.fromFilename(cfs.getSSTablePath(dir.getDirectoryForNewSSTables()));
- try(SSTableTxnWriter writer = SSTableTxnWriter.create(desc, 0, 0, 0))
- {
- writer.append(Util.dk("0"), cf);
- writer.append(Util.dk("1"), cf);
- writer.append(Util.dk("3"), cf);
-
- cfs.addSSTable(writer.closeAndOpenReader());
- }
-
- desc = Descriptor.fromFilename(cfs.getSSTablePath(dir.getDirectoryForNewSSTables()));
- try (SSTableTxnWriter writer = SSTableTxnWriter.create(desc, 0, 0, 0))
- {
- writer.append(Util.dk("0"), cf);
- writer.append(Util.dk("1"), cf);
- writer.append(Util.dk("2"), cf);
- writer.append(Util.dk("3"), cf);
- cfs.addSSTable(writer.closeAndOpenReader());
- }
+ int[] dkays = {0, 1, 2, 3};
+ writeSSTableWithRangeTombstoneMaskingOneColumn(cfs, table, dkays);
Collection<SSTableReader> toCompact = cfs.getLiveSSTables();
assert toCompact.size() == 2;
- // Force compaction on first sstables. Since each row is in only one sstable, we will be using EchoedRow.
Util.compact(cfs, toCompact);
assertEquals(1, cfs.getLiveSSTables().size());
// Now assert we do have the 4 keys
- assertEquals(4, Util.getRangeSlice(cfs).size());
+ assertEquals(4, Util.getAll(Util.cmd(cfs).build()).size());
+
+ ArrayList<DecoratedKey> k = new ArrayList<>();
- ArrayList<DecoratedKey> k = new ArrayList<DecoratedKey>();
- for (Row r : Util.getRangeSlice(cfs))
+ for (FilteredPartition p : Util.getAll(Util.cmd(cfs).build()))
{
- k.add(r.key);
- assertEquals(ByteBufferUtil.bytes("a"),r.cf.getColumn(Util.cellname("a")).value());
- assertNull(r.cf.getColumn(Util.cellname("01")));
- assertEquals(3,r.cf.getColumn(Util.cellname("a")).timestamp());
+ k.add(p.partitionKey());
+ final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
+ try (ReadOrderGroup orderGroup = command.startOrderGroup();
+ PartitionIterator iterator = command.executeInternal(orderGroup))
+ {
+ try (RowIterator rowIterator = iterator.next())
+ {
+ Row row = rowIterator.next();
+ Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("val", false)));
+ assertEquals(ByteBufferUtil.bytes("a"), cell.value());
+ assertEquals(3, cell.timestamp());
+ assertNotSame(ByteBufferUtil.bytes("01"), row.clustering().getRawValues()[0]);
+ assertEquals(ByteBufferUtil.bytes("02"), row.clustering().getRawValues()[0]);
+ }
+ }
}
-
for (SSTableReader sstable : cfs.getLiveSSTables())
{
StatsMetadata stats = sstable.getSSTableMetadata();
- assertEquals(ByteBufferUtil.bytes("0"), stats.minColumnNames.get(0));
- assertEquals(ByteBufferUtil.bytes("b"), stats.maxColumnNames.get(0));
+ assertEquals(ByteBufferUtil.bytes("0"), stats.minClusteringValues.get(0));
+ assertEquals(ByteBufferUtil.bytes("b"), stats.maxClusteringValues.get(0));
}
assertEquals(keys, k);
}
- @Test
- public void testCompactionLog() throws Exception
- {
- SystemKeyspace.discardCompactionsInProgress();
-
- String cf = "Standard4";
- ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
- SchemaLoader.insertData(KEYSPACE1, cf, 0, 1);
- cfs.forceBlockingFlush();
-
- Collection<SSTableReader> sstables = cfs.getLiveSSTables();
- assertFalse(sstables.isEmpty());
- Set<Integer> generations = Sets.newHashSet(Iterables.transform(sstables, new Function<SSTableReader, Integer>()
- {
- public Integer apply(SSTableReader sstable)
- {
- return sstable.descriptor.generation;
- }
- }));
- UUID taskId = SystemKeyspace.startCompaction(cfs, sstables);
- Map<Pair<String, String>, Map<Integer, UUID>> compactionLogs = SystemKeyspace.getUnfinishedCompactions();
- Set<Integer> unfinishedCompactions = compactionLogs.get(Pair.create(KEYSPACE1, cf)).keySet();
- assertTrue(unfinishedCompactions.containsAll(generations));
-
- SystemKeyspace.finishCompaction(taskId);
- compactionLogs = SystemKeyspace.getUnfinishedCompactions();
- assertFalse(compactionLogs.containsKey(Pair.create(KEYSPACE1, cf)));
- }
-
- private void testDontPurgeAccidentaly(String k, String cfname) throws InterruptedException
+ private void testDontPurgeAccidentally(String k, String cfname) throws InterruptedException
{
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+ CFMetaData table = cfs.metadata;
// disable compaction while flushing
cfs.clearUnsafe();
@@ -492,24 +426,24 @@ public class CompactionsTest
// Add test row
DecoratedKey key = Util.dk(k);
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add(cfname, Util.cellname(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0);
- rm.applyUnsafe();
+ RowUpdateBuilder rowUpdateBuilder = new RowUpdateBuilder(table, 0, key);
+ rowUpdateBuilder.clustering("c").add("val", "a");
+ rowUpdateBuilder.build().applyUnsafe();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesBefore = cfs.getLiveSSTables();
- QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
- assertTrue(cfs.getColumnFamily(filter).hasColumns());
+ ImmutableBTreePartition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+ assertTrue(!partition.isEmpty());
+ RowUpdateBuilder deleteRowBuilder = new RowUpdateBuilder(table, 2, key);
+ deleteRowBuilder.clustering("c").delete("val");
+ deleteRowBuilder.build().applyUnsafe();
// Remove key
- rm = new Mutation(KEYSPACE1, key.getKey());
- rm.delete(cfname, 2);
- rm.applyUnsafe();
- ColumnFamily cf = cfs.getColumnFamily(filter);
- assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns());
+ partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
+ assertTrue(partition.iterator().next().cells().iterator().next().isTombstone());
// Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
Thread.sleep(1000);
@@ -524,10 +458,28 @@ public class CompactionsTest
Util.compact(cfs, toCompact);
- cf = cfs.getColumnFamily(filter);
- assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns());
+ SSTableReader newSSTable = null;
+ for (SSTableReader reader : cfs.getLiveSSTables())
+ {
+ assert !toCompact.contains(reader);
+ if (!sstablesBefore.contains(reader))
+ newSSTable = reader;
+ }
+
+ // We cannot read the data, since {@link ReadCommand#withoutPurgeableTombstones} will purge droppable tombstones
+ // but we just want to check here that compaction did *NOT* drop the tombstone, so we read from the SSTable directly
+ // instead
+ ISSTableScanner scanner = newSSTable.getScanner();
+ assertTrue(scanner.hasNext());
+ UnfilteredRowIterator rowIt = scanner.next();
+ assertTrue(rowIt.hasNext());
+ Unfiltered unfiltered = rowIt.next();
+ assertTrue(unfiltered.isRow());
+ Row row = (Row)unfiltered;
+ assertTrue(row.cells().iterator().next().isTombstone());
+ assertFalse(rowIt.hasNext());
+ assertFalse(scanner.hasNext());
}
- */
private static Range<Token> rangeFor(int start, int end)
{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[6/6] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Posted by pa...@apache.org.
Merge branch 'cassandra-3.11' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d3b6a67b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d3b6a67b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d3b6a67b
Branch: refs/heads/trunk
Commit: d3b6a67bbf283e9965ecafa57001fd9dcbb14ba2
Parents: 843a5fd 02e9ddf
Author: Paulo Motta <pa...@gmail.com>
Authored: Wed Jun 6 15:43:37 2018 -0300
Committer: Paulo Motta <pa...@gmail.com>
Committed: Wed Jun 6 15:44:42 2018 -0300
----------------------------------------------------------------------
.../org/apache/cassandra/db/SystemKeyspace.java | 18 +-
.../cassandra/schema/CompactionParams.java | 2 +-
.../db/compaction/LongCompactionsTest.java | 2 +-
.../unit/org/apache/cassandra/SchemaLoader.java | 2 +-
.../AbstractCompactionStrategyTest.java | 2 +-
.../db/compaction/AntiCompactionTest.java | 22 +-
.../CompactionStrategyManagerTest.java | 2 +-
.../db/compaction/CompactionsTest.java | 379 +++++++++----------
.../cassandra/schema/MigrationManagerTest.java | 2 +-
9 files changed, 195 insertions(+), 236 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d3b6a67b/src/java/org/apache/cassandra/db/SystemKeyspace.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/SystemKeyspace.java
index 12ab952,973538f..b4de801
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@@ -101,30 -103,34 +101,30 @@@ public final class SystemKeyspac
public static final String SIZE_ESTIMATES = "size_estimates";
public static final String AVAILABLE_RANGES = "available_ranges";
public static final String TRANSFERRED_RANGES = "transferred_ranges";
- public static final String VIEWS_BUILDS_IN_PROGRESS = "views_builds_in_progress";
+ public static final String TRANSFERRED_RANGES_V2 = "transferred_ranges_v2";
+ public static final String VIEW_BUILDS_IN_PROGRESS = "view_builds_in_progress";
public static final String BUILT_VIEWS = "built_views";
public static final String PREPARED_STATEMENTS = "prepared_statements";
-
- @Deprecated public static final String LEGACY_HINTS = "hints";
- @Deprecated public static final String LEGACY_BATCHLOG = "batchlog";
- @Deprecated public static final String LEGACY_KEYSPACES = "schema_keyspaces";
- @Deprecated public static final String LEGACY_COLUMNFAMILIES = "schema_columnfamilies";
- @Deprecated public static final String LEGACY_COLUMNS = "schema_columns";
- @Deprecated public static final String LEGACY_TRIGGERS = "schema_triggers";
- @Deprecated public static final String LEGACY_USERTYPES = "schema_usertypes";
- @Deprecated public static final String LEGACY_FUNCTIONS = "schema_functions";
- @Deprecated public static final String LEGACY_AGGREGATES = "schema_aggregates";
-
- public static final CFMetaData Batches =
- compile(BATCHES,
- "batches awaiting replay",
- "CREATE TABLE %s ("
- + "id timeuuid,"
- + "mutations list<blob>,"
- + "version int,"
- + "PRIMARY KEY ((id)))")
- .copy(new LocalPartitioner(TimeUUIDType.instance))
- .compaction(CompactionParams.scts(singletonMap("min_threshold", "2")))
- .gcGraceSeconds(0);
-
- private static final CFMetaData Paxos =
- compile(PAXOS,
+ public static final String REPAIRS = "repairs";
+
+ @Deprecated public static final String LEGACY_PEERS = "peers";
+ @Deprecated public static final String LEGACY_PEER_EVENTS = "peer_events";
+ @Deprecated public static final String LEGACY_TRANSFERRED_RANGES = "transferred_ranges";
+
+ public static final TableMetadata Batches =
+ parse(BATCHES,
- "batches awaiting replay",
- "CREATE TABLE %s ("
- + "id timeuuid,"
- + "mutations list<blob>,"
- + "version int,"
- + "PRIMARY KEY ((id)))")
- .partitioner(new LocalPartitioner(TimeUUIDType.instance))
- .compaction(CompactionParams.scts(singletonMap("min_threshold", "2")))
- .build();
++ "batches awaiting replay",
++ "CREATE TABLE %s ("
++ + "id timeuuid,"
++ + "mutations list<blob>,"
++ + "version int,"
++ + "PRIMARY KEY ((id)))")
++ .partitioner(new LocalPartitioner(TimeUUIDType.instance))
++ .compaction(CompactionParams.stcs(singletonMap("min_threshold", "2")))
++ .build();
+
+ private static final TableMetadata Paxos =
+ parse(PAXOS,
"in-progress paxos proposals",
"CREATE TABLE %s ("
+ "row_key blob,"
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d3b6a67b/src/java/org/apache/cassandra/schema/CompactionParams.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/schema/CompactionParams.java
index 1a4a5f1,73271f1..da05667
--- a/src/java/org/apache/cassandra/schema/CompactionParams.java
+++ b/src/java/org/apache/cassandra/schema/CompactionParams.java
@@@ -107,7 -107,7 +107,7 @@@ public final class CompactionParam
return new CompactionParams(klass, allOptions, isEnabled, tombstoneOption);
}
-- public static CompactionParams scts(Map<String, String> options)
++ public static CompactionParams stcs(Map<String, String> options)
{
return create(SizeTieredCompactionStrategy.class, options);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d3b6a67b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
----------------------------------------------------------------------
diff --cc test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
index f204303,d684e11..912b03f
--- a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
@@@ -55,7 -55,7 +55,7 @@@ public class LongCompactionsTes
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)
-- .compaction(CompactionParams.scts(compactionOptions)));
++ .compaction(CompactionParams.stcs(compactionOptions)));
}
@Before
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d3b6a67b/test/unit/org/apache/cassandra/SchemaLoader.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/SchemaLoader.java
index d703bab,567da19..eddaa1e
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@@ -126,31 -119,44 +126,31 @@@ public class SchemaLoade
KeyspaceParams.simple(1),
Tables.of(
// Column Families
- standardCFMD(ks1, "Standard1").compaction(CompactionParams.scts(compactionOptions)).build(),
- standardCFMD(ks1, "Standard1").compaction(CompactionParams.scts(compactionOptions)),
- standardCFMD(ks1, "Standard2"),
- standardCFMD(ks1, "Standard3"),
- standardCFMD(ks1, "Standard4"),
- standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0),
- standardCFMD(ks1, "StandardLong1"),
- standardCFMD(ks1, "StandardLong2"),
- //CFMetaData.Builder.create(ks1, "ValuesWithQuotes").build(),
- superCFMD(ks1, "Super1", LongType.instance),
- superCFMD(ks1, "Super2", LongType.instance),
- superCFMD(ks1, "Super3", LongType.instance),
- superCFMD(ks1, "Super4", UTF8Type.instance),
- superCFMD(ks1, "Super5", bytes),
- superCFMD(ks1, "Super6", LexicalUUIDType.instance, UTF8Type.instance),
- keysIndexCFMD(ks1, "Indexed1", true),
- keysIndexCFMD(ks1, "Indexed2", false),
- //CFMetaData.Builder.create(ks1, "StandardInteger1").withColumnNameComparator(IntegerType.instance).build(),
- //CFMetaData.Builder.create(ks1, "StandardLong3").withColumnNameComparator(IntegerType.instance).build(),
- //CFMetaData.Builder.create(ks1, "Counter1", false, false, true).build(),
- //CFMetaData.Builder.create(ks1, "SuperCounter1", false, false, true, true).build(),
- superCFMD(ks1, "SuperDirectGC", BytesType.instance).gcGraceSeconds(0),
-// jdbcCFMD(ks1, "JdbcInteger", IntegerType.instance).addColumnDefinition(integerColumn(ks1, "JdbcInteger")),
- jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance).addColumnDefinition(utf8Column(ks1, "JdbcUtf8")),
- jdbcCFMD(ks1, "JdbcLong", LongType.instance),
- jdbcCFMD(ks1, "JdbcBytes", bytes),
- jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance),
- //CFMetaData.Builder.create(ks1, "StandardComposite", false, true, false).withColumnNameComparator(composite).build(),
- //CFMetaData.Builder.create(ks1, "StandardComposite2", false, true, false).withColumnNameComparator(compositeMaxMin).build(),
- //CFMetaData.Builder.create(ks1, "StandardDynamicComposite", false, true, false).withColumnNameComparator(dynamicComposite).build(),
- standardCFMD(ks1, "StandardLeveled").compaction(CompactionParams.lcs(leveledOptions)),
- standardCFMD(ks1, "legacyleveled").compaction(CompactionParams.lcs(leveledOptions)),
++ standardCFMD(ks1, "Standard1").compaction(CompactionParams.stcs(compactionOptions)).build(),
+ standardCFMD(ks1, "Standard2").build(),
+ standardCFMD(ks1, "Standard3").build(),
+ standardCFMD(ks1, "Standard4").build(),
+ standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0).build(),
+ standardCFMD(ks1, "StandardLong1").build(),
+ standardCFMD(ks1, "StandardLong2").build(),
+ superCFMD(ks1, "Super1", LongType.instance).build(),
+ superCFMD(ks1, "Super2", LongType.instance).build(),
+ superCFMD(ks1, "Super3", LongType.instance).build(),
+ superCFMD(ks1, "Super4", UTF8Type.instance).build(),
+ superCFMD(ks1, "Super5", bytes).build(),
+ superCFMD(ks1, "Super6", LexicalUUIDType.instance, UTF8Type.instance).build(),
+ keysIndexCFMD(ks1, "Indexed1", true).build(),
+ keysIndexCFMD(ks1, "Indexed2", false).build(),
+ superCFMD(ks1, "SuperDirectGC", BytesType.instance).gcGraceSeconds(0).build(),
+ jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance).addColumn(utf8Column(ks1, "JdbcUtf8")).build(),
+ jdbcCFMD(ks1, "JdbcLong", LongType.instance).build(),
+ jdbcCFMD(ks1, "JdbcBytes", bytes).build(),
+ jdbcCFMD(ks1, "JdbcAscii", AsciiType.instance).build(),
+ standardCFMD(ks1, "StandardLeveled").compaction(CompactionParams.lcs(leveledOptions)).build(),
+ standardCFMD(ks1, "legacyleveled").compaction(CompactionParams.lcs(leveledOptions)).build(),
standardCFMD(ks1, "StandardLowIndexInterval").minIndexInterval(8)
.maxIndexInterval(256)
- .caching(CachingParams.CACHE_NOTHING)
- //CFMetaData.Builder.create(ks1, "UUIDKeys").addPartitionKey("key",UUIDType.instance).build(),
- //CFMetaData.Builder.create(ks1, "MixedTypes").withColumnNameComparator(LongType.instance).addPartitionKey("key", UUIDType.instance).build(),
- //CFMetaData.Builder.create(ks1, "MixedTypesComposite", false, true, false).withColumnNameComparator(composite).addPartitionKey("key", composite).build(),
- //CFMetaData.Builder.create(ks1, "AsciiKeys").addPartitionKey("key", AsciiType.instance).build()
+ .caching(CachingParams.CACHE_NOTHING).build()
)));
// Keyspace 2
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d3b6a67b/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
index e4be57b,481b394..4092f54
--- a/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java
@@@ -59,7 -59,7 +59,7 @@@ public class AbstractCompactionStrategy
SchemaLoader.standardCFMD(KEYSPACE1, LCS_TABLE)
.compaction(CompactionParams.lcs(Collections.emptyMap())),
SchemaLoader.standardCFMD(KEYSPACE1, STCS_TABLE)
-- .compaction(CompactionParams.scts(Collections.emptyMap())),
++ .compaction(CompactionParams.stcs(Collections.emptyMap())),
SchemaLoader.standardCFMD(KEYSPACE1, DTCS_TABLE)
.compaction(CompactionParams.create(DateTieredCompactionStrategy.class, Collections.emptyMap())),
SchemaLoader.standardCFMD(KEYSPACE1, TWCS_TABLE)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d3b6a67b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
index bda05af,ead0349..366c18e
--- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
@@@ -30,19 -27,12 +30,18 @@@ import java.util.UUID
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
-import com.google.common.util.concurrent.RateLimiter;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.After;
- import org.junit.Ignore;
import org.junit.Test;
-import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.MockSchema;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.rows.EncodingStats;
@@@ -57,14 -47,10 +56,15 @@@ import org.apache.cassandra.io.sstable.
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.utils.concurrent.Refs;
import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.utils.concurrent.Transactional;
++import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR;
+import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@@ -165,19 -136,6 +163,18 @@@ public class AntiCompactionTes
}
@Test
+ public void antiCompactOneRepairedAt() throws Exception
+ {
+ antiCompactOne(1000, NO_PENDING_REPAIR);
+ }
+
+ @Test
+ public void antiCompactOnePendingRepair() throws Exception
+ {
+ antiCompactOne(UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID());
+ }
+
- @Ignore
+ @Test
public void antiCompactionSizeTest() throws InterruptedException, IOException
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
@@@ -186,12 -144,12 +183,14 @@@
SSTableReader s = writeFile(cfs, 1000);
cfs.addSSTable(s);
Range<Token> range = new Range<Token>(new BytesToken(ByteBufferUtil.bytes(0)), new BytesToken(ByteBufferUtil.bytes(500)));
++ List<Range<Token>> ranges = Arrays.asList(range);
Collection<SSTableReader> sstables = cfs.getLiveSSTables();
UUID parentRepairSession = UUID.randomUUID();
++ registerParentRepairSession(parentRepairSession, ranges, UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID());
try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION);
Refs<SSTableReader> refs = Refs.ref(sstables))
{
- CompactionManager.instance.performAnticompaction(cfs, Arrays.asList(range), refs, txn, 12345, NO_PENDING_REPAIR, parentRepairSession);
- CompactionManager.instance.performAnticompaction(cfs, Arrays.asList(range), refs, txn, 12345, parentRepairSession);
++ CompactionManager.instance.performAnticompaction(cfs, ranges, refs, txn, 12345, NO_PENDING_REPAIR, parentRepairSession);
}
long sum = 0;
long rows = 0;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d3b6a67b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
index 6f2551d,c654fcd..549ba3f
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
@@@ -78,10 -72,6 +78,10 @@@ public class CompactionStrategyManagerT
* disk assignment based on its generation - See {@link this#getSSTableIndex(Integer[], SSTableReader)}
*/
originalPartitioner = StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
+ SchemaLoader.createKeyspace(KS_PREFIX,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KS_PREFIX, TABLE_PREFIX)
- .compaction(CompactionParams.scts(Collections.emptyMap())));
++ .compaction(CompactionParams.stcs(Collections.emptyMap())));
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d3b6a67b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index 7368598,ad138bf..941ef13
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@@ -18,7 -18,9 +18,13 @@@
*/
package org.apache.cassandra.db.compaction;
- import java.util.*;
+ import java.io.File;
+ import java.nio.ByteBuffer;
-import java.util.*;
++import java.util.ArrayList;
++import java.util.Collection;
++import java.util.HashMap;
++import java.util.Iterator;
++import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.junit.BeforeClass;
@@@ -29,19 -31,37 +35,54 @@@ import org.junit.runner.RunWith
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
- import org.apache.cassandra.schema.TableMetadata;
- import org.apache.cassandra.db.*;
-import org.apache.cassandra.config.CFMetaData;
+ import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.SSTableIterator;
++import org.apache.cassandra.db.Clustering;
++import org.apache.cassandra.db.ColumnFamilyStore;
++import org.apache.cassandra.db.DecoratedKey;
++import org.apache.cassandra.db.DeletionTime;
++import org.apache.cassandra.db.Directories;
++import org.apache.cassandra.db.Keyspace;
++import org.apache.cassandra.db.PartitionPosition;
++import org.apache.cassandra.db.RangeTombstone;
++import org.apache.cassandra.db.ReadExecutionController;
++import org.apache.cassandra.db.RowUpdateBuilder;
++import org.apache.cassandra.db.SinglePartitionReadCommand;
++import org.apache.cassandra.db.Slice;
++import org.apache.cassandra.db.Slices;
+ import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
+ import org.apache.cassandra.db.filter.ColumnFilter;
+ import org.apache.cassandra.db.filter.DataLimits;
+ import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.AsciiType;
+ import org.apache.cassandra.db.partitions.FilteredPartition;
+ import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
+ import org.apache.cassandra.db.partitions.PartitionIterator;
+ import org.apache.cassandra.db.rows.Cell;
-import org.apache.cassandra.db.rows.ColumnData;
+ import org.apache.cassandra.db.rows.Row;
+ import org.apache.cassandra.db.rows.RowIterator;
+ import org.apache.cassandra.db.rows.Unfiltered;
+ import org.apache.cassandra.db.rows.UnfilteredRowIterator;
++import org.apache.cassandra.dht.AbstractBounds;
++import org.apache.cassandra.dht.Bounds;
++import org.apache.cassandra.dht.ByteOrderedPartitioner;
++import org.apache.cassandra.dht.Range;
++import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
--import org.apache.cassandra.dht.*;
+ import org.apache.cassandra.io.sstable.Component;
+ import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+ import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.TableParams;
-import org.apache.cassandra.service.MigrationManager;
+import org.apache.cassandra.schema.MigrationManager;
++import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
--import static org.junit.Assert.*;
++import static org.junit.Assert.assertEquals;
++import static org.junit.Assert.assertFalse;
++import static org.junit.Assert.assertNotEquals;
++import static org.junit.Assert.assertTrue;
@RunWith(OrderedJUnit4ClassRunner.class)
public class CompactionsTest
@@@ -70,9 -90,9 +111,9 @@@
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple(1),
SchemaLoader.denseCFMD(KEYSPACE1, CF_DENSE1)
-- .compaction(CompactionParams.scts(compactionOptions)),
++ .compaction(CompactionParams.stcs(compactionOptions)),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1)
-- .compaction(CompactionParams.scts(compactionOptions)),
++ .compaction(CompactionParams.stcs(compactionOptions)),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD3),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD4),
@@@ -82,7 -102,9 +123,28 @@@
.gcGraceSeconds(0));
}
- public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception
++ public static long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl)
++ {
++ long timestamp = System.currentTimeMillis();
++ TableMetadata cfm = Keyspace.open(ks).getColumnFamilyStore(cf).metadata();
++ for (int i = startRowKey; i <= endRowKey; i++)
++ {
++ DecoratedKey key = Util.dk(Integer.toString(i));
++ for (int j = 0; j < 10; j++)
++ {
++ new RowUpdateBuilder(cfm, timestamp, j > 0 ? ttl : 0, key.getKey())
++ .clustering(Integer.toString(j))
++ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
++ .build()
++ .applyUnsafe();
++ }
++ }
++ return timestamp;
++ }
++
+ // Test to see if sstable has enough expired columns, it is compacted itself.
+ @Test
+ public void testSingleSSTableCompaction() throws Exception
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_DENSE1);
@@@ -116,52 -138,33 +178,14 @@@
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
-
- return store;
- }
-
- public static long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl)
- {
- long timestamp = System.currentTimeMillis();
- TableMetadata cfm = Keyspace.open(ks).getColumnFamilyStore(cf).metadata();
- for (int i = startRowKey; i <= endRowKey; i++)
- {
- DecoratedKey key = Util.dk(Integer.toString(i));
- for (int j = 0; j < 10; j++)
- {
- new RowUpdateBuilder(cfm, timestamp, j > 0 ? ttl : 0, key.getKey())
- .clustering(Integer.toString(j))
- .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
- .build()
- .applyUnsafe();
- }
- }
- return timestamp;
- }
-
- // Test to see if sstable has enough expired columns, it is compacted itself.
- @Test
- public void testSingleSSTableCompactionWithSizeTieredCompaction() throws Exception
- {
- testSingleSSTableCompaction(SizeTieredCompactionStrategy.class.getCanonicalName());
- }
-
- /*
- @Test
- public void testSingleSSTableCompactionWithLeveledCompaction() throws Exception
- {
- ColumnFamilyStore store = testSingleSSTableCompaction(LeveledCompactionStrategy.class.getCanonicalName());
- CompactionStrategyManager strategyManager = store.getCompactionStrategyManager();
- // tombstone removal compaction should not promote level
- assert strategyManager.getSSTableCountPerLevel()[0] == 1;
}
- public static long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl)
- {
- long timestamp = System.currentTimeMillis();
- CFMetaData cfm = Keyspace.open(ks).getColumnFamilyStore(cf).metadata;
- for (int i = startRowKey; i <= endRowKey; i++)
- {
- DecoratedKey key = Util.dk(Integer.toString(i));
- for (int j = 0; j < 10; j++)
- {
- new RowUpdateBuilder(cfm, timestamp, j > 0 ? ttl : 0, key.getKey())
- .clustering(Integer.toString(j))
- .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
- .build()
- .applyUnsafe();
- }
- }
- return timestamp;
- }
-
@Test
public void testSuperColumnTombstones()
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
-- CFMetaData table = cfs.metadata;
++ TableMetadata table = cfs.metadata();
cfs.disableAutoCompaction();
DecoratedKey key = Util.dk("tskey");
@@@ -197,11 -207,9 +228,8 @@@
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
store.clearUnsafe();
- store.metadata.gcGraceSeconds(1);
- store.metadata.compactionStrategyOptions.put("tombstone_compaction_interval", "1");
- store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "false");
- store.reload();
- store.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getName());
+
- MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
- store.reload();
++ MigrationManager.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).compaction(CompactionParams.stcs(compactionOptions)).build(), true);
// disable compaction while flushing
store.disableAutoCompaction();
@@@ -238,13 -246,14 +266,13 @@@
long newSize1 = it.next().uncompressedLength();
long newSize2 = it.next().uncompressedLength();
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize1, newSize1);
+ originalSize1, newSize1);
assertEquals("candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
- originalSize2, newSize2);
+ originalSize2, newSize2);
// now let's enable the magic property
- store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "true");
- store.reload();
+ compactionOptions.put("unchecked_tombstone_compaction", "true");
- MigrationManager.announceColumnFamilyUpdate(store.metadata.params(TableParams.builder(store.metadata.params).gcGraceSeconds(1).compaction(CompactionParams.scts(compactionOptions)).build()), true);
- store.reload();
++ MigrationManager.announceTableUpdate(store.metadata().unbuild().gcGraceSeconds(1).compaction(CompactionParams.stcs(compactionOptions)).build(), true);
//submit background task again and wait for it to complete
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
@@@ -274,68 -282,13 +301,14 @@@
assertEquals(maxTimestampExpected, maxTimestampObserved);
}
- /*
- @Test
- public void testEchoedRow()
- {
- // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653
-
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
-
- // disable compaction while flushing
- cfs.disableAutoCompaction();
-
- // Insert 4 keys in two sstables. We need the sstables to have 2 rows
- // at least to trigger what was causing CASSANDRA-2653
- for (int i=1; i < 5; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
- rm.applyUnsafe();
-
- if (i % 2 == 0)
- cfs.forceBlockingFlush();
- }
- Collection<SSTableReader> toCompact = cfs.getLiveSSTables();
- assertEquals(2, toCompact.size());
-
- // Reinserting the same keys. We will compact only the previous sstable, but we need those new ones
- // to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
- for (int i=1; i < 5; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i));
- Mutation rm = new Mutation(KEYSPACE1, key.getKey());
- rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
- rm.applyUnsafe();
- }
- cfs.forceBlockingFlush();
- SSTableReader tmpSSTable = null;
- for (SSTableReader sstable : cfs.getLiveSSTables())
- if (!toCompact.contains(sstable))
- tmpSSTable = sstable;
- assertNotNull(tmpSSTable);
-
- // Force compaction on first sstables. Since each row is in only one sstable, we will be using EchoedRow.
- Util.compact(cfs, toCompact);
- assertEquals(2, cfs.getLiveSSTables().size());
-
- // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it doesn't hide the problem)
- cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
- assertEquals(1, cfs.getLiveSSTables().size());
-
- // Now assert we do have the 4 keys
- assertEquals(4, Util.getRangeSlice(cfs).size());
- }
+
@Test
- public void testDontPurgeAccidentaly() throws InterruptedException
+ public void testDontPurgeAccidentally() throws InterruptedException
{
- testDontPurgeAccidentaly("test1", "Super5");
- testDontPurgeAccidentally("test1", CF_SUPER5);
++ testDontPurgeAccidentally("test1", "Super5");
// Use CF with gc_grace=0, see last bug of CASSANDRA-2786
- testDontPurgeAccidentaly("test1", "SuperDirectGC");
- testDontPurgeAccidentally("test1", CF_SUPERGC);
++ testDontPurgeAccidentally("test1", "SuperDirectGC");
}
@Test
@@@ -344,6 -297,7 +317,7 @@@
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- CFMetaData table = cfs.metadata;
++ TableMetadata table = cfs.metadata();
// disable compaction while flushing
cfs.disableAutoCompaction();
@@@ -378,6 -331,23 +351,23 @@@
assertEquals( prevGeneration + 1, sstables.iterator().next().descriptor.generation);
}
- public static void writeSSTableWithRangeTombstoneMaskingOneColumn(ColumnFamilyStore cfs, CFMetaData table, int[] dks) {
++ public static void writeSSTableWithRangeTombstoneMaskingOneColumn(ColumnFamilyStore cfs, TableMetadata table, int[] dks) {
+ for (int dk : dks)
+ {
+ RowUpdateBuilder deletedRowUpdateBuilder = new RowUpdateBuilder(table, 1, Util.dk(Integer.toString(dk)));
+ deletedRowUpdateBuilder.clustering("01").add("val", "a"); //Range tombstone covers this (timestamp 2 > 1)
- Clustering startClustering = new Clustering(ByteBufferUtil.bytes("0"));
- Clustering endClustering = new Clustering(ByteBufferUtil.bytes("b"));
++ Clustering startClustering = Clustering.make(ByteBufferUtil.bytes("0"));
++ Clustering endClustering = Clustering.make(ByteBufferUtil.bytes("b"));
+ deletedRowUpdateBuilder.addRangeTombstone(new RangeTombstone(Slice.make(startClustering, endClustering), new DeletionTime(2, (int) (System.currentTimeMillis() / 1000))));
+ deletedRowUpdateBuilder.build().applyUnsafe();
+
+ RowUpdateBuilder notYetDeletedRowUpdateBuilder = new RowUpdateBuilder(table, 3, Util.dk(Integer.toString(dk)));
+ notYetDeletedRowUpdateBuilder.clustering("02").add("val", "a"); //Range tombstone doesn't cover this (timestamp 3 > 2)
+ notYetDeletedRowUpdateBuilder.build().applyUnsafe();
+ }
+ cfs.forceBlockingFlush();
+ }
+
@Test
public void testRangeTombstones()
{
@@@ -388,8 -358,8 +378,8 @@@
// disable compaction while flushing
cfs.disableAutoCompaction();
- final CFMetaData cfmeta = cfs.metadata;
- Directories dir = cfs.directories;
- final CFMetaData table = cfs.metadata;
++ final TableMetadata table = cfs.metadata();
+ Directories dir = cfs.getDirectories();
ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>();
@@@ -431,17 -381,28 +401,28 @@@
assertEquals(1, cfs.getLiveSSTables().size());
// Now assert we do have the 4 keys
- assertEquals(4, Util.getRangeSlice(cfs).size());
+ assertEquals(4, Util.getAll(Util.cmd(cfs).build()).size());
+
+ ArrayList<DecoratedKey> k = new ArrayList<>();
- ArrayList<DecoratedKey> k = new ArrayList<DecoratedKey>();
- for (Row r : Util.getRangeSlice(cfs))
+ for (FilteredPartition p : Util.getAll(Util.cmd(cfs).build()))
{
- k.add(r.key);
- assertEquals(ByteBufferUtil.bytes("a"),r.cf.getColumn(Util.cellname("a")).value());
- assertNull(r.cf.getColumn(Util.cellname("01")));
- assertEquals(3,r.cf.getColumn(Util.cellname("a")).timestamp());
+ k.add(p.partitionKey());
- final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata, FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata), RowFilter.NONE, DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
- try (ReadOrderGroup orderGroup = command.startOrderGroup();
- PartitionIterator iterator = command.executeInternal(orderGroup))
++ final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
++ try (ReadExecutionController executionController = command.executionController();
++ PartitionIterator iterator = command.executeInternal(executionController))
+ {
+ try (RowIterator rowIterator = iterator.next())
+ {
+ Row row = rowIterator.next();
- Cell cell = row.getCell(cfs.metadata.getColumnDefinition(new ColumnIdentifier("val", false)));
++ Cell cell = row.getCell(cfs.metadata().getColumn(new ColumnIdentifier("val", false)));
+ assertEquals(ByteBufferUtil.bytes("a"), cell.value());
+ assertEquals(3, cell.timestamp());
- assertNotSame(ByteBufferUtil.bytes("01"), row.clustering().getRawValues()[0]);
++ assertNotEquals(ByteBufferUtil.bytes("01"), row.clustering().getRawValues()[0]);
+ assertEquals(ByteBufferUtil.bytes("02"), row.clustering().getRawValues()[0]);
+ }
+ }
}
-
for (SSTableReader sstable : cfs.getLiveSSTables())
{
StatsMetadata stats = sstable.getSSTableMetadata();
@@@ -486,6 -418,7 +438,7 @@@
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
- CFMetaData table = cfs.metadata;
++ TableMetadata table = cfs.metadata();
// disable compaction while flushing
cfs.clearUnsafe();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d3b6a67b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
index 455fd23,0000000..3fbc3d7
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
+++ b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java
@@@ -1,577 -1,0 +1,577 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.marshal.ByteType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.locator.OldNetworkTopologyStrategy;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.Util.throwAssert;
+import static org.apache.cassandra.cql3.CQLTester.assertRows;
+import static org.apache.cassandra.cql3.CQLTester.row;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class MigrationManagerTest
+{
+ private static final String KEYSPACE1 = "keyspace1";
+ private static final String KEYSPACE3 = "keyspace3";
+ private static final String KEYSPACE6 = "keyspace6";
+ private static final String EMPTY_KEYSPACE = "test_empty_keyspace";
+ private static final String TABLE1 = "standard1";
+ private static final String TABLE2 = "standard2";
+ private static final String TABLE1i = "indexed1";
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.startGossiper();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, TABLE1),
+ SchemaLoader.standardCFMD(KEYSPACE1, TABLE2));
+ SchemaLoader.createKeyspace(KEYSPACE3,
+ KeyspaceParams.simple(5),
+ SchemaLoader.standardCFMD(KEYSPACE1, TABLE1),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE3, TABLE1i, true));
+ SchemaLoader.createKeyspace(KEYSPACE6,
+ KeyspaceParams.simple(1),
+ SchemaLoader.compositeIndexCFMD(KEYSPACE6, TABLE1i, true));
+ }
+
+ @Test
+ public void testTableMetadataBuilder() throws ConfigurationException
+ {
+ TableMetadata.Builder builder =
+ TableMetadata.builder(KEYSPACE1, "TestApplyCFM_CF")
+ .addPartitionKeyColumn("keys", BytesType.instance)
+ .addClusteringColumn("col", BytesType.instance)
+ .comment("No comment")
+ .gcGraceSeconds(100000)
- .compaction(CompactionParams.scts(ImmutableMap.of("min_threshold", "500", "max_threshold", "500")));
++ .compaction(CompactionParams.stcs(ImmutableMap.of("min_threshold", "500", "max_threshold", "500")));
+
+ for (int i = 0; i < 5; i++)
+ {
+ ByteBuffer name = ByteBuffer.wrap(new byte[] { (byte)i });
+ builder.addRegularColumn(ColumnIdentifier.getInterned(name, BytesType.instance), ByteType.instance);
+ }
+
+
+ TableMetadata table = builder.build();
+ // we'll be adding this one later. make sure it's not already there.
+ assertNull(table.getColumn(ByteBuffer.wrap(new byte[]{ 5 })));
+
+ // add one.
+ ColumnMetadata addIndexDef = ColumnMetadata.regularColumn(table, ByteBuffer.wrap(new byte[] { 5 }), BytesType.instance);
+ builder.addColumn(addIndexDef);
+
+ // remove one.
+ ColumnMetadata removeIndexDef = ColumnMetadata.regularColumn(table, ByteBuffer.wrap(new byte[] { 0 }), BytesType.instance);
+ builder.removeRegularOrStaticColumn(removeIndexDef.name);
+
+ TableMetadata table2 = builder.build();
+
+ for (int i = 1; i < table2.columns().size(); i++)
+ assertNotNull(table2.getColumn(ByteBuffer.wrap(new byte[]{ 1 })));
+ assertNull(table2.getColumn(ByteBuffer.wrap(new byte[]{ 0 })));
+ assertNotNull(table2.getColumn(ByteBuffer.wrap(new byte[]{ 5 })));
+ }
+
+ @Test
+ public void testInvalidNames()
+ {
+ String[] valid = {"1", "a", "_1", "b_", "__", "1_a"};
+ for (String s : valid)
+ assertTrue(SchemaConstants.isValidName(s));
+
+ String[] invalid = {"b@t", "dash-y", "", " ", "dot.s", ".hidden"};
+ for (String s : invalid)
+ assertFalse(SchemaConstants.isValidName(s));
+ }
+
+ @Test
+ public void addNewCfToBogusKeyspace()
+ {
+ TableMetadata newCf = addTestTable("MadeUpKeyspace", "NewCF", "new cf");
+ try
+ {
+ MigrationManager.announceNewTable(newCf);
+ throw new AssertionError("You shouldn't be able to do anything to a keyspace that doesn't exist.");
+ }
+ catch (ConfigurationException expected)
+ {
+ }
+ }
+
+ @Test
+ public void addNewTable() throws ConfigurationException
+ {
+ final String ksName = KEYSPACE1;
+ final String tableName = "anewtable";
+ KeyspaceMetadata original = Schema.instance.getKeyspaceMetadata(ksName);
+
+ TableMetadata cfm = addTestTable(original.name, tableName, "A New Table");
+
+ assertFalse(Schema.instance.getKeyspaceMetadata(ksName).tables.get(cfm.name).isPresent());
+ MigrationManager.announceNewTable(cfm);
+
+ assertTrue(Schema.instance.getKeyspaceMetadata(ksName).tables.get(cfm.name).isPresent());
+ assertEquals(cfm, Schema.instance.getKeyspaceMetadata(ksName).tables.get(cfm.name).get());
+
+ // now read and write to it.
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, col, val) VALUES (?, ?, ?)",
+ ksName, tableName),
+ "key0", "col0", "val0");
+
+ // flush to exercise more than just hitting the memtable
+ ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(tableName);
+ assertNotNull(cfs);
+ cfs.forceBlockingFlush();
+
+ // and make sure we get out what we put in
+ UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", ksName, tableName));
+ assertRows(rows, row("key0", "col0", "val0"));
+ }
+
+ @Test
+ public void dropCf() throws ConfigurationException
+ {
+ // sanity
+ final KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(KEYSPACE1);
+ assertNotNull(ks);
+ final TableMetadata cfm = ks.tables.getNullable(TABLE1);
+ assertNotNull(cfm);
+
+ // write some data, force a flush, then verify that files exist on disk.
+ for (int i = 0; i < 100; i++)
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE1, TABLE1),
+ "dropCf", "col" + i, "anyvalue");
+ ColumnFamilyStore store = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name);
+ assertNotNull(store);
+ store.forceBlockingFlush();
+ assertTrue(store.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().size() > 0);
+
+ MigrationManager.announceTableDrop(ks.name, cfm.name);
+
+ assertFalse(Schema.instance.getKeyspaceMetadata(ks.name).tables.get(cfm.name).isPresent());
+
+ // any write should fail.
+ boolean success = true;
+ try
+ {
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE1, TABLE1),
+ "dropCf", "col0", "anyvalue");
+ }
+ catch (Throwable th)
+ {
+ success = false;
+ }
+ assertFalse("This mutation should have failed since the CF no longer exists.", success);
+
+ // verify that the files are gone.
+ Supplier<Object> lambda = () -> {
+ for (File file : store.getDirectories().sstableLister(Directories.OnTxnErr.THROW).listFiles())
+ {
+ if (file.getPath().endsWith("Data.db") && !new File(file.getPath().replace("Data.db", "Compacted")).exists())
+ return false;
+ }
+ return true;
+ };
+ Util.spinAssertEquals(true, lambda, 30);
+
+ }
+
+ @Test
+ public void addNewKS() throws ConfigurationException
+ {
+ TableMetadata cfm = addTestTable("newkeyspace1", "newstandard1", "A new cf for a new ks");
+ KeyspaceMetadata newKs = KeyspaceMetadata.create(cfm.keyspace, KeyspaceParams.simple(5), Tables.of(cfm));
+ MigrationManager.announceNewKeyspace(newKs);
+
+ assertNotNull(Schema.instance.getKeyspaceMetadata(cfm.keyspace));
+ assertEquals(Schema.instance.getKeyspaceMetadata(cfm.keyspace), newKs);
+
+ // test reads and writes.
+ QueryProcessor.executeInternal("INSERT INTO newkeyspace1.newstandard1 (key, col, val) VALUES (?, ?, ?)",
+ "key0", "col0", "val0");
+ ColumnFamilyStore store = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name);
+ assertNotNull(store);
+ store.forceBlockingFlush();
+
+ UntypedResultSet rows = QueryProcessor.executeInternal("SELECT * FROM newkeyspace1.newstandard1");
+ assertRows(rows, row("key0", "col0", "val0"));
+ }
+
+ @Test
+ public void dropKS() throws ConfigurationException
+ {
+ // sanity
+ final KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(KEYSPACE1);
+ assertNotNull(ks);
+ final TableMetadata cfm = ks.tables.getNullable(TABLE2);
+ assertNotNull(cfm);
+
+ // write some data, force a flush, then verify that files exist on disk.
+ for (int i = 0; i < 100; i++)
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE1, TABLE2),
+ "dropKs", "col" + i, "anyvalue");
+ ColumnFamilyStore cfs = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name);
+ assertNotNull(cfs);
+ cfs.forceBlockingFlush();
+ assertTrue(!cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().isEmpty());
+
+ MigrationManager.announceKeyspaceDrop(ks.name);
+
+ assertNull(Schema.instance.getKeyspaceMetadata(ks.name));
+
+ // write should fail.
+ boolean success = true;
+ try
+ {
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE1, TABLE2),
+ "dropKs", "col0", "anyvalue");
+ }
+ catch (Throwable th)
+ {
+ success = false;
+ }
+ assertFalse("This mutation should have failed since the KS no longer exists.", success);
+
+ // reads should fail too.
+ boolean threw = false;
+ try
+ {
+ Keyspace.open(ks.name);
+ }
+ catch (Throwable th)
+ {
+ threw = true;
+ }
+ assertTrue(threw);
+ }
+
+ @Test
+ public void dropKSUnflushed() throws ConfigurationException
+ {
+ // sanity
+ final KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(KEYSPACE3);
+ assertNotNull(ks);
+ final TableMetadata cfm = ks.tables.getNullable(TABLE1);
+ assertNotNull(cfm);
+
+ // write some data
+ for (int i = 0; i < 100; i++)
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, name, val) VALUES (?, ?, ?)",
+ KEYSPACE3, TABLE1),
+ "dropKs", "col" + i, "anyvalue");
+
+ MigrationManager.announceKeyspaceDrop(ks.name);
+
+ assertNull(Schema.instance.getKeyspaceMetadata(ks.name));
+ }
+
+ @Test
+ public void createEmptyKsAddNewCf() throws ConfigurationException
+ {
+ assertNull(Schema.instance.getKeyspaceMetadata(EMPTY_KEYSPACE));
+ KeyspaceMetadata newKs = KeyspaceMetadata.create(EMPTY_KEYSPACE, KeyspaceParams.simple(5));
+ MigrationManager.announceNewKeyspace(newKs);
+ assertNotNull(Schema.instance.getKeyspaceMetadata(EMPTY_KEYSPACE));
+
+ String tableName = "added_later";
+ TableMetadata newCf = addTestTable(EMPTY_KEYSPACE, tableName, "A new CF to add to an empty KS");
+
+ //should not exist until apply
+ assertFalse(Schema.instance.getKeyspaceMetadata(newKs.name).tables.get(newCf.name).isPresent());
+
+ //add the new CF to the empty space
+ MigrationManager.announceNewTable(newCf);
+
+ assertTrue(Schema.instance.getKeyspaceMetadata(newKs.name).tables.get(newCf.name).isPresent());
+ assertEquals(Schema.instance.getKeyspaceMetadata(newKs.name).tables.get(newCf.name).get(), newCf);
+
+ // now read and write to it.
+ QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, col, val) VALUES (?, ?, ?)",
+ EMPTY_KEYSPACE, tableName),
+ "key0", "col0", "val0");
+
+ ColumnFamilyStore cfs = Keyspace.open(newKs.name).getColumnFamilyStore(newCf.name);
+ assertNotNull(cfs);
+ cfs.forceBlockingFlush();
+
+ UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", EMPTY_KEYSPACE, tableName));
+ assertRows(rows, row("key0", "col0", "val0"));
+ }
+
+ @Test
+ public void testUpdateKeyspace() throws ConfigurationException
+ {
+ // create a keyspace to serve as existing.
+ TableMetadata cf = addTestTable("UpdatedKeyspace", "AddedStandard1", "A new cf for a new ks");
+ KeyspaceMetadata oldKs = KeyspaceMetadata.create(cf.keyspace, KeyspaceParams.simple(5), Tables.of(cf));
+
+ MigrationManager.announceNewKeyspace(oldKs);
+
+ assertNotNull(Schema.instance.getKeyspaceMetadata(cf.keyspace));
+ assertEquals(Schema.instance.getKeyspaceMetadata(cf.keyspace), oldKs);
+
+ // names should match.
+ KeyspaceMetadata newBadKs2 = KeyspaceMetadata.create(cf.keyspace + "trash", KeyspaceParams.simple(4));
+ try
+ {
+ MigrationManager.announceKeyspaceUpdate(newBadKs2);
+ throw new AssertionError("Should not have been able to update a KS with an invalid KS name.");
+ }
+ catch (ConfigurationException ex)
+ {
+ // expected.
+ }
+
+ Map<String, String> replicationMap = new HashMap<>();
+ replicationMap.put(ReplicationParams.CLASS, OldNetworkTopologyStrategy.class.getName());
+ replicationMap.put("replication_factor", "1");
+
+ KeyspaceMetadata newKs = KeyspaceMetadata.create(cf.keyspace, KeyspaceParams.create(true, replicationMap));
+ MigrationManager.announceKeyspaceUpdate(newKs);
+
+ KeyspaceMetadata newFetchedKs = Schema.instance.getKeyspaceMetadata(newKs.name);
+ assertEquals(newFetchedKs.params.replication.klass, newKs.params.replication.klass);
+ assertFalse(newFetchedKs.params.replication.klass.equals(oldKs.params.replication.klass));
+ }
+
+ /*
+ @Test
+ public void testUpdateColumnFamilyNoIndexes() throws ConfigurationException
+ {
+ // create a keyspace with a cf to update.
+ CFMetaData cf = addTestTable("UpdatedCfKs", "Standard1added", "A new cf that will be updated");
+ KSMetaData ksm = KSMetaData.testMetadata(cf.ksName, SimpleStrategy.class, KSMetaData.optsWithRF(1), cf);
+ MigrationManager.announceNewKeyspace(ksm);
+
+ assertNotNull(Schema.instance.getKSMetaData(cf.ksName));
+ assertEquals(Schema.instance.getKSMetaData(cf.ksName), ksm);
+ assertNotNull(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName));
+
+ // updating certain fields should fail.
+ CFMetaData newCfm = cf.copy();
+ newCfm.defaultValidator(BytesType.instance);
+ newCfm.minCompactionThreshold(5);
+ newCfm.maxCompactionThreshold(31);
+
+ // test valid operations.
+ newCfm.comment("Modified comment");
+ MigrationManager.announceTableUpdate(newCfm); // doesn't get set back here.
+
+ newCfm.readRepairChance(0.23);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ newCfm.gcGraceSeconds(12);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ newCfm.defaultValidator(UTF8Type.instance);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ newCfm.minCompactionThreshold(3);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ newCfm.maxCompactionThreshold(33);
+ MigrationManager.announceTableUpdate(newCfm);
+
+ // can't test changing the reconciler because there is only one impl.
+
+ // check the cumulative affect.
+ assertEquals(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getComment(), newCfm.getComment());
+ assertEquals(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getReadRepairChance(), newCfm.getReadRepairChance(), 0.0001);
+ assertEquals(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getGcGraceSeconds(), newCfm.getGcGraceSeconds());
+ assertEquals(UTF8Type.instance, Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getDefaultValidator());
+
+ // Change tableId
+ newCfm = new CFMetaData(cf.ksName, cf.cfName, cf.cfType, cf.comparator);
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blown up when you used a different id.");
+ }
+ catch (ConfigurationException expected) {}
+
+ // Change cfName
+ newCfm = new CFMetaData(cf.ksName, cf.cfName + "_renamed", cf.cfType, cf.comparator);
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blown up when you used a different name.");
+ }
+ catch (ConfigurationException expected) {}
+
+ // Change ksName
+ newCfm = new CFMetaData(cf.ksName + "_renamed", cf.cfName, cf.cfType, cf.comparator);
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blown up when you used a different keyspace.");
+ }
+ catch (ConfigurationException expected) {}
+
+ // Change cf type
+ newCfm = new CFMetaData(cf.ksName, cf.cfName, ColumnFamilyType.Super, cf.comparator);
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blwon up when you used a different cf type.");
+ }
+ catch (ConfigurationException expected) {}
+
+ // Change comparator
+ newCfm = new CFMetaData(cf.ksName, cf.cfName, cf.cfType, new SimpleDenseCellNameType(TimeUUIDType.instance));
+ CFMetaData.copyOpts(newCfm, cf);
+ try
+ {
+ cf.apply(newCfm);
+ throw new AssertionError("Should have blown up when you used a different comparator.");
+ }
+ catch (ConfigurationException expected) {}
+ }
+ */
+
+ @Test
+ public void testDropIndex() throws ConfigurationException
+ {
+ // persist keyspace definition in the system keyspace
+ SchemaKeyspace.makeCreateKeyspaceMutation(Schema.instance.getKeyspaceMetadata(KEYSPACE6), FBUtilities.timestampMicros()).build().applyUnsafe();
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE6).getColumnFamilyStore(TABLE1i);
+ String indexName = TABLE1i + "_birthdate_key_index";
+
+ // insert some data. save the sstable descriptor so we can make sure it's marked for delete after the drop
+ QueryProcessor.executeInternal(String.format(
+ "INSERT INTO %s.%s (key, c1, birthdate, notbirthdate) VALUES (?, ?, ?, ?)",
+ KEYSPACE6,
+ TABLE1i),
+ "key0", "col0", 1L, 1L);
+
+ cfs.forceBlockingFlush();
+ ColumnFamilyStore indexCfs = cfs.indexManager.getIndexByName(indexName)
+ .getBackingTable()
+ .orElseThrow(throwAssert("Cannot access index cfs"));
+ Descriptor desc = indexCfs.getLiveSSTables().iterator().next().descriptor;
+
+ // drop the index
+ TableMetadata meta = cfs.metadata();
+ IndexMetadata existing = meta.indexes
+ .get(indexName)
+ .orElseThrow(throwAssert("Index not found"));
+
+ MigrationManager.announceTableUpdate(meta.unbuild().indexes(meta.indexes.without(existing.name)).build());
+
+ // check
+ assertTrue(cfs.indexManager.listIndexes().isEmpty());
+ LifecycleTransaction.waitForDeletions();
+ assertFalse(new File(desc.filenameFor(Component.DATA)).exists());
+ }
+
+ @Test
+ public void testValidateNullKeyspace() throws Exception
+ {
+ TableMetadata.Builder builder = TableMetadata.builder(null, TABLE1).addPartitionKeyColumn("partitionKey", BytesType.instance);
+
+ TableMetadata table1 = builder.build();
+ thrown.expect(ConfigurationException.class);
+ thrown.expectMessage(null + "." + TABLE1 + ": Keyspace name must not be empty");
+ table1.validate();
+ }
+
+ @Test
+ public void testValidateCompatibilityIDMismatch() throws Exception
+ {
+ TableMetadata.Builder builder = TableMetadata.builder(KEYSPACE1, TABLE1).addPartitionKeyColumn("partitionKey", BytesType.instance);
+
+ TableMetadata table1 = builder.build();
+ TableMetadata table2 = table1.unbuild().id(TableId.generate()).build();
+ thrown.expect(ConfigurationException.class);
+ thrown.expectMessage(KEYSPACE1 + "." + TABLE1 + ": Table ID mismatch");
+ table1.validateCompatibility(table2);
+ }
+
+ @Test
+ public void testValidateCompatibilityNameMismatch() throws Exception
+ {
+ TableMetadata.Builder builder1 = TableMetadata.builder(KEYSPACE1, TABLE1).addPartitionKeyColumn("partitionKey", BytesType.instance);
+ TableMetadata.Builder builder2 = TableMetadata.builder(KEYSPACE1, TABLE2).addPartitionKeyColumn("partitionKey", BytesType.instance);
+ TableMetadata table1 = builder1.build();
+ TableMetadata table2 = builder2.build();
+ thrown.expect(ConfigurationException.class);
+ thrown.expectMessage(KEYSPACE1 + "." + TABLE1 + ": Table mismatch");
+ table1.validateCompatibility(table2);
+ }
+
+ private TableMetadata addTestTable(String ks, String cf, String comment)
+ {
+ return
+ TableMetadata.builder(ks, cf)
+ .addPartitionKeyColumn("key", UTF8Type.instance)
+ .addClusteringColumn("col", UTF8Type.instance)
+ .addRegularColumn("val", UTF8Type.instance)
+ .comment(comment)
+ .build();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org