You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by pa...@apache.org on 2017/09/25 06:30:21 UTC
[6/8] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3e3d56ec/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
index a31b9a1,0000000..84a20de
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
@@@ -1,389 -1,0 +1,390 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Function;
+
+import com.google.common.collect.Iterables;
+import org.junit.Test;
+
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class GcCompactionTest extends CQLTester
+{
+ static final int KEY_COUNT = 10;
+ static final int CLUSTERING_COUNT = 20;
+
+ // Test needs synchronous table drop to avoid flushes causing flaky failures
+
+ @Override
+ protected String createTable(String query)
+ {
+ return super.createTable(KEYSPACE_PER_TEST, query);
+ }
+
+ @Override
+ protected UntypedResultSet execute(String query, Object... values) throws Throwable
+ {
+ return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values);
+ }
+
+ @Override
+ public ColumnFamilyStore getCurrentColumnFamilyStore()
+ {
+ return super.getCurrentColumnFamilyStore(KEYSPACE_PER_TEST);
+ }
+
+ public void flush()
+ {
+ flush(KEYSPACE_PER_TEST);
+ }
+
+ @Test
+ public void testGcCompactionPartitions() throws Throwable
+ {
+ runCompactionTest("CREATE TABLE %s(" +
+ " key int," +
+ " column int," +
+ " data int," +
+ " extra text," +
+ " PRIMARY KEY((key, column), data)" +
+ ") WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones' : 'row' };"
+ );
+
+ }
+
+ @Test
+ public void testGcCompactionRows() throws Throwable
+ {
+ runCompactionTest("CREATE TABLE %s(" +
+ " key int," +
+ " column int," +
+ " data int," +
+ " extra text," +
+ " PRIMARY KEY(key, column)" +
+ ") WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones' : 'row' };"
+ );
+
+ }
+
+ @Test
+ public void testGcCompactionRanges() throws Throwable
+ {
+
+ runCompactionTest("CREATE TABLE %s(" +
+ " key int," +
+ " column int," +
+ " col2 int," +
+ " data int," +
+ " extra text," +
+ " PRIMARY KEY(key, column, data)" +
+ ") WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones' : 'row' };"
+ );
+ }
+
+ private void runCompactionTest(String tableDef) throws Throwable
+ {
+ createTable(tableDef);
+
+ for (int i = 0; i < KEY_COUNT; ++i)
+ for (int j = 0; j < CLUSTERING_COUNT; ++j)
+ execute("INSERT INTO %s (key, column, data, extra) VALUES (?, ?, ?, ?)", i, j, i+j, "" + i + ":" + j);
+
+ Set<SSTableReader> readers = new HashSet<>();
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+
+ flush();
+ assertEquals(1, cfs.getLiveSSTables().size());
+ SSTableReader table0 = getNewTable(readers);
+ assertEquals(0, countTombstoneMarkers(table0));
+ int rowCount = countRows(table0);
+
+ deleteWithSomeInserts(3, 5, 10);
+ flush();
+ assertEquals(2, cfs.getLiveSSTables().size());
+ SSTableReader table1 = getNewTable(readers);
+ assertTrue(countRows(table1) > 0);
+ assertTrue(countTombstoneMarkers(table1) > 0);
+
+ deleteWithSomeInserts(5, 6, 0);
+ flush();
+ assertEquals(3, cfs.getLiveSSTables().size());
+ SSTableReader table2 = getNewTable(readers);
+ assertEquals(0, countRows(table2));
+ assertTrue(countTombstoneMarkers(table2) > 0);
+
+ CompactionManager.instance.forceUserDefinedCompaction(table0.getFilename());
+
+ assertEquals(3, cfs.getLiveSSTables().size());
+ SSTableReader table3 = getNewTable(readers);
+ assertEquals(0, countTombstoneMarkers(table3));
+ assertTrue(rowCount > countRows(table3));
+ }
+
+ @Test
+ public void testGcCompactionCells() throws Throwable
+ {
+ createTable("CREATE TABLE %s(" +
+ " key int," +
+ " column int," +
+ " data int," +
+ " extra text," +
+ " PRIMARY KEY(key)" +
+ ") WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones' : 'cell' };"
+ );
+
+ for (int i = 0; i < KEY_COUNT; ++i)
+ for (int j = 0; j < CLUSTERING_COUNT; ++j)
+ execute("INSERT INTO %s (key, column, data, extra) VALUES (?, ?, ?, ?)", i, j, i+j, "" + i + ":" + j);
+
+ Set<SSTableReader> readers = new HashSet<>();
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+
+ flush();
+ assertEquals(1, cfs.getLiveSSTables().size());
+ SSTableReader table0 = getNewTable(readers);
+ assertEquals(0, countTombstoneMarkers(table0));
+ int cellCount = countCells(table0);
+
+ deleteWithSomeInserts(3, 0, 2);
+ flush();
+ assertEquals(2, cfs.getLiveSSTables().size());
+ SSTableReader table1 = getNewTable(readers);
+ assertTrue(countCells(table1) > 0);
+ assertEquals(0, countTombstoneMarkers(table0));
+
+ CompactionManager.instance.forceUserDefinedCompaction(table0.getFilename());
+
+ assertEquals(2, cfs.getLiveSSTables().size());
+ SSTableReader table3 = getNewTable(readers);
+ assertEquals(0, countTombstoneMarkers(table3));
+ assertTrue(cellCount > countCells(table3));
+ }
+
+ @Test
+ public void testGcCompactionStatic() throws Throwable
+ {
+ createTable("CREATE TABLE %s(" +
+ " key int," +
+ " column int," +
+ " data int static," +
+ " extra text," +
+ " PRIMARY KEY(key, column)" +
+ ") WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones' : 'cell' };"
+ );
+
+ for (int i = 0; i < KEY_COUNT; ++i)
+ for (int j = 0; j < CLUSTERING_COUNT; ++j)
+ execute("INSERT INTO %s (key, column, data, extra) VALUES (?, ?, ?, ?)", i, j, i+j, "" + i + ":" + j);
+
+ Set<SSTableReader> readers = new HashSet<>();
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+
+ flush();
+ assertEquals(1, cfs.getLiveSSTables().size());
+ SSTableReader table0 = getNewTable(readers);
+ assertEquals(0, countTombstoneMarkers(table0));
+ int cellCount = countStaticCells(table0);
+ assertEquals(KEY_COUNT, cellCount);
+
+ execute("DELETE data FROM %s WHERE key = 0"); // delete static cell
+ execute("INSERT INTO %s (key, data) VALUES (1, 0)"); // overwrite static cell
+ flush();
+ assertEquals(2, cfs.getLiveSSTables().size());
+ SSTableReader table1 = getNewTable(readers);
+ assertTrue(countStaticCells(table1) > 0);
+ assertEquals(0, countTombstoneMarkers(table0));
+
+ CompactionManager.instance.forceUserDefinedCompaction(table0.getFilename());
+
+ assertEquals(2, cfs.getLiveSSTables().size());
+ SSTableReader table3 = getNewTable(readers);
+ assertEquals(0, countTombstoneMarkers(table3));
+ assertEquals(cellCount - 2, countStaticCells(table3));
+ }
+
+ @Test
+ public void testGcCompactionComplexColumn() throws Throwable
+ {
+ createTable("CREATE TABLE %s(" +
+ " key int," +
+ " data map<int, int>," +
+ " extra text," +
+ " PRIMARY KEY(key)" +
+ ") WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones' : 'cell' };"
+ );
+
+ for (int i = 0; i < KEY_COUNT; ++i)
+ for (int j = 0; j < CLUSTERING_COUNT; ++j)
+ execute("UPDATE %s SET data[?] = ? WHERE key = ?", j, i+j, i);
+
+ Set<SSTableReader> readers = new HashSet<>();
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+
+ flush();
+ assertEquals(1, cfs.getLiveSSTables().size());
+ SSTableReader table0 = getNewTable(readers);
+ assertEquals(0, countTombstoneMarkers(table0));
+ int cellCount = countComplexCells(table0);
+
+ deleteWithSomeInsertsComplexColumn(3, 5, 8);
+ flush();
+ assertEquals(2, cfs.getLiveSSTables().size());
+ SSTableReader table1 = getNewTable(readers);
+ assertTrue(countComplexCells(table1) > 0);
+ assertEquals(0, countTombstoneMarkers(table0));
+
+ CompactionManager.instance.forceUserDefinedCompaction(table0.getFilename());
+
+ assertEquals(2, cfs.getLiveSSTables().size());
+ SSTableReader table3 = getNewTable(readers);
+ assertEquals(0, countTombstoneMarkers(table3));
+ assertEquals(cellCount - 23, countComplexCells(table3));
+ }
+
+ @Test
+ public void testLocalDeletionTime() throws Throwable
+ {
+ createTable("create table %s (k int, c1 int, primary key (k, c1)) with compaction = {'class': 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones':'row'}");
+ execute("delete from %s where k = 1");
+ Set<SSTableReader> readers = new HashSet<>(getCurrentColumnFamilyStore().getLiveSSTables());
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ SSTableReader oldSSTable = getNewTable(readers);
+ Thread.sleep(2000);
+ execute("delete from %s where k = 1");
+ getCurrentColumnFamilyStore().forceBlockingFlush();
+ SSTableReader newTable = getNewTable(readers);
+
+ CompactionManager.instance.forceUserDefinedCompaction(oldSSTable.getFilename());
+
+ // Old table now doesn't contain any data and should disappear.
+ assertEquals(Collections.singleton(newTable), getCurrentColumnFamilyStore().getLiveSSTables());
+ }
+
+ private SSTableReader getNewTable(Set<SSTableReader> readers)
+ {
+ Set<SSTableReader> newOnes = new HashSet<>(getCurrentColumnFamilyStore().getLiveSSTables());
+ newOnes.removeAll(readers);
+ assertEquals(1, newOnes.size());
+ readers.addAll(newOnes);
+ return Iterables.get(newOnes, 0);
+ }
+
+ void deleteWithSomeInserts(int key_step, int delete_step, int readd_step) throws Throwable
+ {
+ for (int i = 0; i < KEY_COUNT; i += key_step)
+ {
+ if (delete_step > 0)
+ for (int j = i % delete_step; j < CLUSTERING_COUNT; j += delete_step)
+ {
+ execute("DELETE FROM %s WHERE key = ? AND column = ?", i, j);
+ }
+ if (readd_step > 0)
+ for (int j = i % readd_step; j < CLUSTERING_COUNT; j += readd_step)
+ {
+ execute("INSERT INTO %s (key, column, data, extra) VALUES (?, ?, ?, ?)", i, j, i-j, "readded " + i + ":" + j);
+ }
+ }
+ }
+
+ void deleteWithSomeInsertsComplexColumn(int key_step, int delete_step, int readd_step) throws Throwable
+ {
+ for (int i = 0; i < KEY_COUNT; i += key_step)
+ {
+ if (delete_step > 0)
+ for (int j = i % delete_step; j < CLUSTERING_COUNT; j += delete_step)
+ {
+ execute("DELETE data[?] FROM %s WHERE key = ?", j, i);
+ }
+ if (readd_step > 0)
+ for (int j = i % readd_step; j < CLUSTERING_COUNT; j += readd_step)
+ {
+ execute("UPDATE %s SET data[?] = ? WHERE key = ?", j, -(i+j), i);
+ }
+ }
+ }
+
+ int countTombstoneMarkers(SSTableReader reader)
+ {
+ int nowInSec = FBUtilities.nowInSeconds();
+ return count(reader, x -> x.isRangeTombstoneMarker() || x.isRow() && ((Row) x).hasDeletion(nowInSec) ? 1 : 0, x -> x.partitionLevelDeletion().isLive() ? 0 : 1);
+ }
+
+ int countRows(SSTableReader reader)
+ {
++ boolean enforceStrictLiveness = reader.metadata.enforceStrictLiveness();
+ int nowInSec = FBUtilities.nowInSeconds();
- return count(reader, x -> x.isRow() && ((Row) x).hasLiveData(nowInSec) ? 1 : 0, x -> 0);
++ return count(reader, x -> x.isRow() && ((Row) x).hasLiveData(nowInSec, enforceStrictLiveness) ? 1 : 0, x -> 0);
+ }
+
+ int countCells(SSTableReader reader)
+ {
+ return count(reader, x -> x.isRow() ? Iterables.size((Row) x) : 0, x -> 0);
+ }
+
+ int countStaticCells(SSTableReader reader)
+ {
+ return count(reader, x -> 0, x -> Iterables.size(x.staticRow()));
+ }
+
+ int countComplexCells(SSTableReader reader)
+ {
+ return count(reader, x -> x.isRow() ? ((Row) x).stream().mapToInt(this::countComplex).sum() : 0, x -> 0);
+ }
+
+ int countComplex(ColumnData c)
+ {
+ if (!(c instanceof ComplexColumnData))
+ return 0;
+ ComplexColumnData ccd = (ComplexColumnData) c;
+ return ccd.cellsCount();
+ }
+
+ int count(SSTableReader reader, Function<Unfiltered, Integer> predicate, Function<UnfilteredRowIterator, Integer> partitionPredicate)
+ {
+ int instances = 0;
+ try (ISSTableScanner partitions = reader.getScanner())
+ {
+ while (partitions.hasNext())
+ {
+ try (UnfilteredRowIterator iter = partitions.next())
+ {
+ instances += partitionPredicate.apply(iter);
+ while (iter.hasNext())
+ {
+ Unfiltered atom = iter.next();
+ instances += predicate.apply(atom);
+ }
+ }
+ }
+ }
+ return instances;
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3e3d56ec/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3e3d56ec/test/unit/org/apache/cassandra/cql3/ViewTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3e3d56ec/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
index 9120546,967a85c..363ef72
--- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
+++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
@@@ -112,17 -113,21 +113,21 @@@ public class RangeTombstoneTes
int nowInSec = FBUtilities.nowInSeconds();
for (int i : live)
- assertTrue("Row " + i + " should be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec));
+ assertTrue("Row " + i + " should be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
for (int i : dead)
- assertFalse("Row " + i + " shouldn't be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec));
+ assertFalse("Row " + i + " shouldn't be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
// Queries by slices
partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).fromIncl(7).toIncl(30).build());
for (int i : new int[]{ 7, 8, 9, 11, 13, 15, 17, 28, 29, 30 })
- assertTrue("Row " + i + " should be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec));
+ assertTrue("Row " + i + " should be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
for (int i : new int[]{ 10, 12, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 })
- assertFalse("Row " + i + " shouldn't be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec));
+ assertFalse("Row " + i + " shouldn't be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
}
@Test
@@@ -385,7 -390,8 +390,7 @@@
CompactionManager.instance.disableAutoCompaction();
Keyspace keyspace = Keyspace.open(KSNAME);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CFNAME);
-
+ boolean enforceStrictLiveness = cfs.metadata.enforceStrictLiveness();
-
// Inserting data
String key = "k2";
@@@ -408,22 -414,30 +413,30 @@@
int nowInSec = FBUtilities.nowInSeconds();
for (int i = 0; i < 5; i++)
- assertTrue("Row " + i + " should be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec));
+ assertTrue("Row " + i + " should be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
for (int i = 16; i < 20; i++)
- assertTrue("Row " + i + " should be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec));
+ assertTrue("Row " + i + " should be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
for (int i = 5; i <= 15; i++)
- assertFalse("Row " + i + " shouldn't be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec));
+ assertFalse("Row " + i + " shouldn't be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
// Compact everything and re-test
CompactionManager.instance.performMaximal(cfs, false);
partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build());
for (int i = 0; i < 5; i++)
- assertTrue("Row " + i + " should be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(FBUtilities.nowInSeconds()));
+ assertTrue("Row " + i + " should be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(FBUtilities.nowInSeconds(),
- enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(FBUtilities.nowInSeconds(),
++ enforceStrictLiveness));
for (int i = 16; i < 20; i++)
- assertTrue("Row " + i + " should be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(FBUtilities.nowInSeconds()));
+ assertTrue("Row " + i + " should be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(FBUtilities.nowInSeconds(),
- enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(FBUtilities.nowInSeconds(),
++ enforceStrictLiveness));
for (int i = 5; i <= 15; i++)
- assertFalse("Row " + i + " shouldn't be live", partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec));
+ assertFalse("Row " + i + " shouldn't be live",
- partition.getRow(new Clustering(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
++ partition.getRow(Clustering.make(bb(i))).hasLiveData(nowInSec, enforceStrictLiveness));
}
@Test
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3e3d56ec/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org