You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2012/09/21 17:06:20 UTC
git commit: move testStandardColumnCompactions and
testSuperColumnCompactions to LongSompactionsTest
Updated Branches:
refs/heads/trunk 98eb543a9 -> f92fb2241
move testStandardColumnCompactions and testSuperColumnCompactions to LongSompactionsTest
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f92fb224
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f92fb224
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f92fb224
Branch: refs/heads/trunk
Commit: f92fb2241fc2ea5c7fb25ee885337404de3fbb49
Parents: 98eb543
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Sep 21 10:05:51 2012 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Sep 21 10:05:51 2012 -0500
----------------------------------------------------------------------
.../db/compaction/LongCompactionSpeedTest.java | 102 -------
.../db/compaction/LongCompactionsTest.java | 217 +++++++++++++++
.../cassandra/db/compaction/CompactionsTest.java | 107 +-------
3 files changed, 218 insertions(+), 208 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f92fb224/test/long/org/apache/cassandra/db/compaction/LongCompactionSpeedTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionSpeedTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionSpeedTest.java
deleted file mode 100644
index 8338c2c..0000000
--- a/test/long/org/apache/cassandra/db/compaction/LongCompactionSpeedTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-package org.apache.cassandra.db.compaction;
-
-import java.util.*;
-
-import org.apache.cassandra.config.Schema;
-import org.junit.Test;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.Util;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableUtils;
-
-public class LongCompactionSpeedTest extends SchemaLoader
-{
- public static final String TABLE1 = "Keyspace1";
-
- /**
- * Test compaction with a very wide row.
- */
- @Test
- public void testCompactionWide() throws Exception
- {
- testCompaction(2, 1, 200000);
- }
-
- /**
- * Test compaction with lots of skinny rows.
- */
- @Test
- public void testCompactionSlim() throws Exception
- {
- testCompaction(2, 200000, 1);
- }
-
- /**
- * Test compaction with lots of small sstables.
- */
- @Test
- public void testCompactionMany() throws Exception
- {
- testCompaction(100, 800, 5);
- }
-
- protected void testCompaction(int sstableCount, int rowsPerSSTable, int colsPerRow) throws Exception
- {
- CompactionManager.instance.disableAutoCompaction();
-
- Table table = Table.open(TABLE1);
- ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
-
- ArrayList<SSTableReader> sstables = new ArrayList<SSTableReader>();
- for (int k = 0; k < sstableCount; k++)
- {
- SortedMap<String,ColumnFamily> rows = new TreeMap<String,ColumnFamily>();
- for (int j = 0; j < rowsPerSSTable; j++)
- {
- String key = String.valueOf(j);
- IColumn[] cols = new IColumn[colsPerRow];
- for (int i = 0; i < colsPerRow; i++)
- {
- // last sstable has highest timestamps
- cols[i] = Util.column(String.valueOf(i), String.valueOf(i), k);
- }
- rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols));
- }
- SSTableReader sstable = SSTableUtils.prepare().write(rows);
- sstables.add(sstable);
- store.addSSTable(sstable);
- }
-
- // give garbage collection a bit of time to catch up
- Thread.sleep(1000);
-
- long start = System.currentTimeMillis();
- final int gcBefore = (int) (System.currentTimeMillis() / 1000) - Schema.instance.getCFMetaData(TABLE1, "Standard1").getGcGraceSeconds();
- new CompactionTask(store, sstables, gcBefore).execute(null);
- System.out.println(String.format("%s: sstables=%d rowsper=%d colsper=%d: %d ms",
- this.getClass().getName(),
- sstableCount,
- rowsPerSSTable,
- colsPerRow,
- System.currentTimeMillis() - start));
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f92fb224/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
new file mode 100644
index 0000000..4407170
--- /dev/null
+++ b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
@@ -0,0 +1,217 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.cassandra.db.compaction;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.Schema;
+import org.junit.Test;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.SSTableUtils;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static junit.framework.Assert.assertEquals;
+
+public class LongCompactionsTest extends SchemaLoader
+{
+ public static final String TABLE1 = "Keyspace1";
+
+ /**
+ * Test compaction with a very wide row.
+ */
+ @Test
+ public void testCompactionWide() throws Exception
+ {
+ testCompaction(2, 1, 200000);
+ }
+
+ /**
+ * Test compaction with lots of skinny rows.
+ */
+ @Test
+ public void testCompactionSlim() throws Exception
+ {
+ testCompaction(2, 200000, 1);
+ }
+
+ /**
+ * Test compaction with lots of small sstables.
+ */
+ @Test
+ public void testCompactionMany() throws Exception
+ {
+ testCompaction(100, 800, 5);
+ }
+
+ protected void testCompaction(int sstableCount, int rowsPerSSTable, int colsPerRow) throws Exception
+ {
+ CompactionManager.instance.disableAutoCompaction();
+
+ Table table = Table.open(TABLE1);
+ ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
+
+ ArrayList<SSTableReader> sstables = new ArrayList<SSTableReader>();
+ for (int k = 0; k < sstableCount; k++)
+ {
+ SortedMap<String,ColumnFamily> rows = new TreeMap<String,ColumnFamily>();
+ for (int j = 0; j < rowsPerSSTable; j++)
+ {
+ String key = String.valueOf(j);
+ IColumn[] cols = new IColumn[colsPerRow];
+ for (int i = 0; i < colsPerRow; i++)
+ {
+ // last sstable has highest timestamps
+ cols[i] = Util.column(String.valueOf(i), String.valueOf(i), k);
+ }
+ rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols));
+ }
+ SSTableReader sstable = SSTableUtils.prepare().write(rows);
+ sstables.add(sstable);
+ store.addSSTable(sstable);
+ }
+
+ // give garbage collection a bit of time to catch up
+ Thread.sleep(1000);
+
+ long start = System.currentTimeMillis();
+ final int gcBefore = (int) (System.currentTimeMillis() / 1000) - Schema.instance.getCFMetaData(TABLE1, "Standard1").getGcGraceSeconds();
+ new CompactionTask(store, sstables, gcBefore).execute(null);
+ System.out.println(String.format("%s: sstables=%d rowsper=%d colsper=%d: %d ms",
+ this.getClass().getName(),
+ sstableCount,
+ rowsPerSSTable,
+ colsPerRow,
+ System.currentTimeMillis() - start));
+ }
+
+ @Test
+ public void testStandardColumnCompactions() throws IOException, ExecutionException, InterruptedException
+ {
+ // this test does enough rows to force multiple block indexes to be used
+ Table table = Table.open(TABLE1);
+ ColumnFamilyStore cfs = table.getColumnFamilyStore("Standard1");
+
+ final int ROWS_PER_SSTABLE = 10;
+ final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE;
+
+ // disable compaction while flushing
+ cfs.disableAutoCompaction();
+
+ long maxTimestampExpected = Long.MIN_VALUE;
+ Set<DecoratedKey> inserted = new HashSet<DecoratedKey>();
+ for (int j = 0; j < SSTABLES; j++) {
+ for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
+ DecoratedKey key = Util.dk(String.valueOf(i % 2));
+ RowMutation rm = new RowMutation(TABLE1, key.key);
+ long timestamp = j * ROWS_PER_SSTABLE + i;
+ rm.add(new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))),
+ ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ timestamp);
+ maxTimestampExpected = Math.max(timestamp, maxTimestampExpected);
+ rm.apply();
+ inserted.add(key);
+ }
+ cfs.forceBlockingFlush();
+ CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected);
+ assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs).size());
+ }
+
+ forceCompactions(cfs);
+
+ assertEquals(inserted.size(), Util.getRangeSlice(cfs).size());
+
+ // make sure max timestamp of compacted sstables is recorded properly after compaction.
+ CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected);
+ cfs.truncate();
+ }
+
+ @Test
+ public void testSuperColumnCompactions() throws IOException, ExecutionException, InterruptedException
+ {
+ Table table = Table.open(TABLE1);
+ ColumnFamilyStore cfs = table.getColumnFamilyStore("Super1");
+
+ final int ROWS_PER_SSTABLE = 10;
+ final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE;
+
+ //disable compaction while flushing
+ cfs.disableAutoCompaction();
+
+ long maxTimestampExpected = Long.MIN_VALUE;
+ Set<DecoratedKey> inserted = new HashSet<DecoratedKey>();
+ ByteBuffer superColumn = ByteBufferUtil.bytes("TestSuperColumn");
+ for (int j = 0; j < SSTABLES; j++)
+ {
+ for (int i = 0; i < ROWS_PER_SSTABLE; i++)
+ {
+ DecoratedKey key = Util.dk(String.valueOf(i % 2));
+ RowMutation rm = new RowMutation(TABLE1, key.key);
+ long timestamp = j * ROWS_PER_SSTABLE + i;
+ rm.add(new QueryPath("Super1", superColumn, ByteBufferUtil.bytes((long)(i / 2))),
+ ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ timestamp);
+ maxTimestampExpected = Math.max(timestamp, maxTimestampExpected);
+ rm.apply();
+ inserted.add(key);
+ }
+ cfs.forceBlockingFlush();
+ CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected);
+ assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs, superColumn).size());
+ }
+
+ forceCompactions(cfs);
+
+ assertEquals(inserted.size(), Util.getRangeSlice(cfs, superColumn).size());
+
+ // make sure max timestamp of compacted sstables is recorded properly after compaction.
+ CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected);
+ }
+
+ private void forceCompactions(ColumnFamilyStore cfs) throws ExecutionException, InterruptedException
+ {
+ // re-enable compaction with thresholds low enough to force a few rounds
+ cfs.setCompactionThresholds(2, 4);
+
+ // loop submitting parallel compactions until they all return 0
+ do
+ {
+ ArrayList<Future<?>> compactions = new ArrayList<Future<?>>();
+ for (int i = 0; i < 10; i++)
+ compactions.add(CompactionManager.instance.submitBackground(cfs));
+ // another compaction attempt will be launched in the background by
+ // each completing compaction: not much we can do to control them here
+ FBUtilities.waitOnFutures(compactions);
+ } while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0);
+
+ if (cfs.getSSTables().size() > 1)
+ {
+ CompactionManager.instance.performMaximal(cfs);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f92fb224/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index dc5a256..6795f87 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -62,47 +62,6 @@ public class CompactionsTest extends SchemaLoader
testBlacklisting(LeveledCompactionStrategy.class.getCanonicalName());
}
- @Test
- public void testStandardColumnCompactions() throws IOException, ExecutionException, InterruptedException
- {
- // this test does enough rows to force multiple block indexes to be used
- Table table = Table.open(TABLE1);
- ColumnFamilyStore cfs = table.getColumnFamilyStore("Standard1");
-
- final int ROWS_PER_SSTABLE = 10;
- final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE;
-
- // disable compaction while flushing
- cfs.disableAutoCompaction();
-
- long maxTimestampExpected = Long.MIN_VALUE;
- Set<DecoratedKey> inserted = new HashSet<DecoratedKey>();
- for (int j = 0; j < SSTABLES; j++) {
- for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
- DecoratedKey key = Util.dk(String.valueOf(i % 2));
- RowMutation rm = new RowMutation(TABLE1, key.key);
- long timestamp = j * ROWS_PER_SSTABLE + i;
- rm.add(new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))),
- ByteBufferUtil.EMPTY_BYTE_BUFFER,
- timestamp);
- maxTimestampExpected = Math.max(timestamp, maxTimestampExpected);
- rm.apply();
- inserted.add(key);
- }
- cfs.forceBlockingFlush();
- assertMaxTimestamp(cfs, maxTimestampExpected);
- assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs).size());
- }
-
- forceCompactions(cfs);
-
- assertEquals(inserted.size(), Util.getRangeSlice(cfs).size());
-
- // make sure max timestamp of compacted sstables is recorded properly after compaction.
- assertMaxTimestamp(cfs, maxTimestampExpected);
- cfs.truncate();
- }
-
/**
* Test to see if sstable has enough expired columns, it is compacted itself.
*/
@@ -205,48 +164,6 @@ public class CompactionsTest extends SchemaLoader
}
@Test
- public void testSuperColumnCompactions() throws IOException, ExecutionException, InterruptedException
- {
- Table table = Table.open(TABLE1);
- ColumnFamilyStore cfs = table.getColumnFamilyStore("Super1");
-
- final int ROWS_PER_SSTABLE = 10;
- final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE;
-
- //disable compaction while flushing
- cfs.disableAutoCompaction();
-
- long maxTimestampExpected = Long.MIN_VALUE;
- Set<DecoratedKey> inserted = new HashSet<DecoratedKey>();
- ByteBuffer superColumn = ByteBufferUtil.bytes("TestSuperColumn");
- for (int j = 0; j < SSTABLES; j++)
- {
- for (int i = 0; i < ROWS_PER_SSTABLE; i++)
- {
- DecoratedKey key = Util.dk(String.valueOf(i % 2));
- RowMutation rm = new RowMutation(TABLE1, key.key);
- long timestamp = j * ROWS_PER_SSTABLE + i;
- rm.add(new QueryPath("Super1", superColumn, ByteBufferUtil.bytes((long)(i / 2))),
- ByteBufferUtil.EMPTY_BYTE_BUFFER,
- timestamp);
- maxTimestampExpected = Math.max(timestamp, maxTimestampExpected);
- rm.apply();
- inserted.add(key);
- }
- cfs.forceBlockingFlush();
- assertMaxTimestamp(cfs, maxTimestampExpected);
- assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(cfs, superColumn).size());
- }
-
- forceCompactions(cfs);
-
- assertEquals(inserted.size(), Util.getRangeSlice(cfs, superColumn).size());
-
- // make sure max timestamp of compacted sstables is recorded properly after compaction.
- assertMaxTimestamp(cfs, maxTimestampExpected);
- }
-
- @Test
public void testSuperColumnTombstones() throws IOException, ExecutionException, InterruptedException
{
Table table = Table.open(TABLE1);
@@ -283,7 +200,7 @@ public class CompactionsTest extends SchemaLoader
assert sc.getSubColumns().isEmpty();
}
- public void assertMaxTimestamp(ColumnFamilyStore cfs, long maxTimestampExpected)
+ public static void assertMaxTimestamp(ColumnFamilyStore cfs, long maxTimestampExpected)
{
long maxTimestampObserved = Long.MIN_VALUE;
for (SSTableReader sstable : cfs.getSSTables())
@@ -291,28 +208,6 @@ public class CompactionsTest extends SchemaLoader
assertEquals(maxTimestampExpected, maxTimestampObserved);
}
- private void forceCompactions(ColumnFamilyStore cfs) throws ExecutionException, InterruptedException
- {
- // re-enable compaction with thresholds low enough to force a few rounds
- cfs.setCompactionThresholds(2, 4);
-
- // loop submitting parallel compactions until they all return 0
- do
- {
- ArrayList<Future<?>> compactions = new ArrayList<Future<?>>();
- for (int i = 0; i < 10; i++)
- compactions.add(CompactionManager.instance.submitBackground(cfs));
- // another compaction attempt will be launched in the background by
- // each completing compaction: not much we can do to control them here
- FBUtilities.waitOnFutures(compactions);
- } while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0);
-
- if (cfs.getSSTables().size() > 1)
- {
- CompactionManager.instance.performMaximal(cfs);
- }
- }
-
@Test
public void testEchoedRow() throws IOException, ExecutionException, InterruptedException
{