You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yu...@apache.org on 2017/02/23 02:14:23 UTC
[02/10] cassandra git commit: run
LongLeveledCompactionStrategyTest.testLeveledScanner in a separate table
run LongLeveledCompactionStrategyTest.testLeveledScanner in a separate table
patch by marcuse; reviewed by yukim for CASSANDRA-12202
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6ffd5cc5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6ffd5cc5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6ffd5cc5
Branch: refs/heads/cassandra-3.0
Commit: 6ffd5cc5d28658b4d058de79a9bea9c10a82c8d4
Parents: 88f36a0
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jul 14 15:31:30 2016 +0200
Committer: Yuki Morishita <yu...@apache.org>
Committed: Thu Feb 23 10:35:50 2017 +0900
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../LongLeveledCompactionStrategyTest.java | 84 +++++++++++++-------
.../unit/org/apache/cassandra/SchemaLoader.java | 3 +
.../LeveledCompactionStrategyTest.java | 2 +-
4 files changed, 62 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ffd5cc5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1a22814..033b366 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.10
+ * Fix flaky LongLeveledCompactionStrategyTest (CASSANDRA-12202)
* Fix failing COPY TO STDOUT (CASSANDRA-12497)
* Fix ColumnCounter::countAll behaviour for reverse queries (CASSANDRA-13222)
* Exceptions encountered calling getSeeds() breaks OTC thread (CASSANDRA-13018)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ffd5cc5/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
index 5439a72..8e63006 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
@@ -25,6 +25,7 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
@@ -41,6 +42,7 @@ public class LongLeveledCompactionStrategyTest
{
public static final String KEYSPACE1 = "LongLeveledCompactionStrategyTest";
public static final String CF_STANDARDLVL = "StandardLeveled";
+ public static final String CF_STANDARDLVL2 = "StandardLeveled2";
@BeforeClass
public static void defineSchema() throws ConfigurationException
@@ -53,6 +55,9 @@ public class LongLeveledCompactionStrategyTest
KSMetaData.optsWithRF(1),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLVL)
.compactionStrategyClass(LeveledCompactionStrategy.class)
+ .compactionStrategyOptions(leveledOptions),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLVL2)
+ .compactionStrategyClass(LeveledCompactionStrategy.class)
.compactionStrategyOptions(leveledOptions));
}
@@ -144,16 +149,34 @@ public class LongLeveledCompactionStrategyTest
@Test
public void testLeveledScanner() throws Exception
{
- testParallelLeveledCompaction();
Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARDLVL);
- store.disableAutoCompaction();
-
+ final ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARDLVL2);
WrappingCompactionStrategy strategy = ((WrappingCompactionStrategy) store.getCompactionStrategy());
- LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategy.getWrappedStrategies().get(1);
+ final LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategy.getWrappedStrategies().get(1);
- ByteBuffer value = ByteBuffer.wrap(new byte[10 * 1024]); // 10 KB value
+ ByteBuffer value = ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files
+
+ // Enough data to have a level 1 and 2
+ int rows = 128;
+ int columns = 10;
+ // Adds enough data to trigger multiple sstable per level
+ for (int r = 0; r < rows; r++)
+ {
+ DecoratedKey key = Util.dk(String.valueOf(r));
+ Mutation rm = new Mutation(KEYSPACE1, key.getKey());
+ for (int c = 0; c < columns; c++)
+ {
+ rm.add(CF_STANDARDLVL2, Util.cellname("column" + c), value, 0);
+ }
+ rm.apply();
+ store.forceBlockingFlush();
+ }
+
+ value = ByteBuffer.wrap(new byte[10 * 1024]); // 10 KB value
+ LeveledCompactionStrategyTest.waitForLeveling(store);
+ // wait for higher-level compactions to finish
+ store.disableAutoCompaction();
// Adds 10 partitions
for (int r = 0; r < 10; r++)
{
@@ -161,7 +184,7 @@ public class LongLeveledCompactionStrategyTest
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
for (int c = 0; c < 10; c++)
{
- rm.add(CF_STANDARDLVL, Util.cellname("column" + c), value, 0);
+ rm.add(CF_STANDARDLVL2, Util.cellname("column" + c), value, 0);
}
rm.apply();
}
@@ -169,33 +192,40 @@ public class LongLeveledCompactionStrategyTest
//Flush sstable
store.forceBlockingFlush();
- Collection<SSTableReader> allSSTables = store.getSSTables();
- for (SSTableReader sstable : allSSTables)
+ store.runWithCompactionsDisabled(new Callable<Void>()
{
- if (sstable.getSSTableLevel() == 0)
+ public Void call() throws Exception
{
- System.out.println("Mutating L0-SSTABLE level to L1 to simulate a bug: " + sstable.getFilename());
- sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 1);
- sstable.reloadSSTableMetadata();
- }
- }
+ Collection<SSTableReader> allSSTables = store.getSSTables();
+ for (SSTableReader sstable : allSSTables)
+ {
+ if (sstable.getSSTableLevel() == 0)
+ {
+ System.out.println("Mutating L0-SSTABLE level to L1 to simulate a bug: " + sstable.getFilename());
+ sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 1);
+ sstable.reloadSSTableMetadata();
+ }
+ }
- try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(allSSTables))
- {
- //Verify that leveled scanners will always iterate in ascending order (CASSANDRA-9935)
- for (ISSTableScanner scanner : scannerList.scanners)
- {
- DecoratedKey lastKey = null;
- while (scanner.hasNext())
+ try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(allSSTables))
{
- OnDiskAtomIterator row = scanner.next();
- if (lastKey != null)
+ //Verify that leveled scanners will always iterate in ascending order (CASSANDRA-9935)
+ for (ISSTableScanner scanner : scannerList.scanners)
{
- assertTrue("row " + row.getKey() + " received out of order wrt " + lastKey, row.getKey().compareTo(lastKey) >= 0);
+ DecoratedKey lastKey = null;
+ while (scanner.hasNext())
+ {
+ OnDiskAtomIterator row = scanner.next();
+ if (lastKey != null)
+ {
+ assertTrue("row " + row.getKey() + " received out of order wrt " + lastKey, row.getKey().compareTo(lastKey) >= 0);
+ }
+ lastKey = row.getKey();
+ }
}
- lastKey = row.getKey();
}
+ return null;
}
- }
+ }, true);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ffd5cc5/test/unit/org/apache/cassandra/SchemaLoader.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index 2048f74..45748a9 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -180,6 +180,9 @@ public class SchemaLoader
standardCFMD(ks1, "StandardLeveled")
.compactionStrategyClass(LeveledCompactionStrategy.class)
.compactionStrategyOptions(leveledOptions),
+ standardCFMD(ks1, "StandardLeveled2")
+ .compactionStrategyClass(LeveledCompactionStrategy.class)
+ .compactionStrategyOptions(leveledOptions),
standardCFMD(ks1, "legacyleveled")
.compactionStrategyClass(LeveledCompactionStrategy.class)
.compactionStrategyOptions(leveledOptions),
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6ffd5cc5/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
index 8b9ca08..0047678 100644
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
@@ -207,7 +207,7 @@ public class LeveledCompactionStrategyTest
/**
* wait for leveled compaction to quiesce on the given columnfamily
*/
- private void waitForLeveling(ColumnFamilyStore cfs) throws InterruptedException
+ public static void waitForLeveling(ColumnFamilyStore cfs) throws InterruptedException
{
WrappingCompactionStrategy strategyManager = (WrappingCompactionStrategy)cfs.getCompactionStrategy();
while (true)