You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2015/04/24 10:42:55 UTC
cassandra git commit: Make CompactionAwareWriterTest work with
compression
Repository: cassandra
Updated Branches:
refs/heads/trunk 5023e620d -> e87008bb9
Make CompactionAwareWriterTest work with compression
Patch by marcuse; reviewed by pcmanus for CASSANDRA-9225
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e87008bb
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e87008bb
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e87008bb
Branch: refs/heads/trunk
Commit: e87008bb9cd37315759623bfba0e3f8ba1eddca1
Parents: 5023e62
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Apr 24 10:38:02 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Fri Apr 24 10:38:02 2015 +0200
----------------------------------------------------------------------
.../SplittingSizeTieredCompactionWriter.java | 2 +-
.../compaction/CompactionAwareWriterTest.java | 59 ++++++++++++++------
2 files changed, 42 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e87008bb/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
index 18c58ea..c97270c 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
@@ -46,7 +46,7 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter
{
private static final Logger logger = LoggerFactory.getLogger(SplittingSizeTieredCompactionWriter.class);
- private static final long DEFAULT_SMALLEST_SSTABLE_BYTES = 50_000_000;
+ public static final long DEFAULT_SMALLEST_SSTABLE_BYTES = 50_000_000;
private final double[] ratios;
private final SSTableRewriter sstableWriter;
private final long totalSize;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e87008bb/test/unit/org/apache/cassandra/db/compaction/CompactionAwareWriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAwareWriterTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAwareWriterTest.java
index ac12491..88074af 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionAwareWriterTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAwareWriterTest.java
@@ -17,17 +17,12 @@
*/
package org.apache.cassandra.db.compaction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
import com.google.common.primitives.Longs;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -68,6 +63,16 @@ public class CompactionAwareWriterTest
SchemaLoader.standardCFMD(KEYSPACE1, CF));
}
+
+ @Before
+ public void clear()
+ {
+ // avoid one test affecting the next one
+ Keyspace ks = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = ks.getColumnFamilyStore(CF);
+ cfs.clearUnsafe();
+ }
+
@Test
public void testDefaultCompactionWriter()
{
@@ -97,8 +102,8 @@ public class CompactionAwareWriterTest
populate(cfs, rowCount);
Set<SSTableReader> sstables = new HashSet<>(cfs.getSSTables());
long beforeSize = sstables.iterator().next().onDiskLength();
- int sstableCount = (int)beforeSize/10;
- CompactionAwareWriter writer = new MaxSSTableSizeWriter(cfs, sstables, sstables, sstableCount, 0, false, OperationType.COMPACTION);
+ int sstableSize = (int)beforeSize/10;
+ CompactionAwareWriter writer = new MaxSSTableSizeWriter(cfs, sstables, sstables, sstableSize, 0, false, OperationType.COMPACTION);
int rows = compact(cfs, sstables, writer);
assertEquals(10, cfs.getSSTables().size());
assertEquals(rowCount, rows);
@@ -130,7 +135,9 @@ public class CompactionAwareWriterTest
});
for (SSTableReader sstable : sortedSSTables)
{
- assertEquals(expectedSize, sstable.onDiskLength(), 10000);
+ // we dont create smaller files than this, everything will be in the last file
+ if (expectedSize > SplittingSizeTieredCompactionWriter.DEFAULT_SMALLEST_SSTABLE_BYTES)
+ assertEquals(expectedSize, sstable.onDiskLength(), expectedSize / 100); // allow 1% diff in estimated vs actual size
expectedSize /= 2;
}
assertEquals(rowCount, rows);
@@ -144,14 +151,15 @@ public class CompactionAwareWriterTest
Keyspace ks = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = ks.getColumnFamilyStore(CF);
cfs.disableAutoCompaction();
- int rowCount = 10000;
+ int rowCount = 20000;
+ int targetSSTableCount = 50;
populate(cfs, rowCount);
Set<SSTableReader> sstables = new HashSet<>(cfs.getSSTables());
long beforeSize = sstables.iterator().next().onDiskLength();
- int sstableCount = (int)beforeSize/100;
- CompactionAwareWriter writer = new MajorLeveledCompactionWriter(cfs, sstables, sstables, sstableCount, false, OperationType.COMPACTION);
+ int sstableSize = (int)beforeSize/targetSSTableCount;
+ CompactionAwareWriter writer = new MajorLeveledCompactionWriter(cfs, sstables, sstables, sstableSize, false, OperationType.COMPACTION);
int rows = compact(cfs, sstables, writer);
- assertEquals(100, cfs.getSSTables().size());
+ assertEquals(targetSSTableCount, cfs.getSSTables().size());
int [] levelCounts = new int[5];
assertEquals(rowCount, rows);
for (SSTableReader sstable : cfs.getSSTables())
@@ -160,7 +168,7 @@ public class CompactionAwareWriterTest
}
assertEquals(0, levelCounts[0]);
assertEquals(10, levelCounts[1]);
- assertEquals(90, levelCounts[2]);
+ assertEquals(targetSSTableCount - 10, levelCounts[2]); // note that if we want more levels, fix this
for (int i = 3; i < levelCounts.length; i++)
assertEquals(0, levelCounts[i]);
validateData(cfs, rowCount);
@@ -190,17 +198,32 @@ public class CompactionAwareWriterTest
private void populate(ColumnFamilyStore cfs, int count)
{
long timestamp = System.currentTimeMillis();
+ byte [] payload = new byte[1000];
+ new Random().nextBytes(payload);
+ ByteBuffer b = ByteBuffer.wrap(payload);
for (int i = 0; i < count; i++)
{
DecoratedKey key = Util.dk(Integer.toString(i));
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
for (int j = 0; j < 10; j++)
rm.add(CF, Util.cellname(Integer.toString(j)),
- ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ b,
timestamp);
rm.applyUnsafe();
}
cfs.forceBlockingFlush();
+ if (cfs.getSSTables().size() > 1)
+ {
+ // we want just one big sstable to avoid doing actual compaction in compact() above
+ try
+ {
+ cfs.forceMajorCompaction();
+ }
+ catch (Throwable t)
+ {
+ throw new RuntimeException(t);
+ }
+ }
assert cfs.getSSTables().size() == 1 : cfs.getSSTables();
}
private void validateData(ColumnFamilyStore cfs, int rowCount)