You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2014/12/19 14:22:53 UTC
[1/2] cassandra git commit: Run major compactions for
repaired/unrepaired in parallel
Repository: cassandra
Updated Branches:
refs/heads/trunk 382834d4e -> 31099a167
Run major compactions for repaired/unrepaired in parallel
Patch by marcuse; reviewed by belliottsmith for CASSANDRA-8510
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c45ed171
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c45ed171
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c45ed171
Branch: refs/heads/trunk
Commit: c45ed1714eaa1536f8c72d355d2831e4aa04a53b
Parents: a7edaa5
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Dec 19 08:10:11 2014 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Fri Dec 19 14:20:33 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 27 ++++++++++++--------
.../db/compaction/CompactionsPurgeTest.java | 10 +++-----
3 files changed, 22 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c45ed171/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bdfa397..e5a8f05 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.3
+ * Run major compactions for repaired/unrepaired in parallel (CASSANDRA-8510)
* (cqlsh) Fix compression options in DESCRIBE TABLE output when compression
is disabled (CASSANDRA-8288)
* (cqlsh) Fix DESCRIBE output after keyspaces are altered (CASSANDRA-7623)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c45ed171/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 3977d9c..9f5951c 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -450,26 +450,33 @@ public class CompactionManager implements CompactionManagerMBean
public void performMaximal(final ColumnFamilyStore cfStore) throws InterruptedException, ExecutionException
{
- submitMaximal(cfStore, getDefaultGcBefore(cfStore)).get();
+ FBUtilities.waitOnFutures(submitMaximal(cfStore, getDefaultGcBefore(cfStore)));
}
- public Future<?> submitMaximal(final ColumnFamilyStore cfStore, final int gcBefore)
+ public List<Future<?>> submitMaximal(final ColumnFamilyStore cfStore, final int gcBefore)
{
// here we compute the task off the compaction executor, so having that present doesn't
// confuse runWithCompactionsDisabled -- i.e., we don't want to deadlock ourselves, waiting
// for ourselves to finish/acknowledge cancellation before continuing.
final Collection<AbstractCompactionTask> tasks = cfStore.getCompactionStrategy().getMaximalTask(gcBefore);
- Runnable runnable = new WrappedRunnable()
+
+ if (tasks == null)
+ return Collections.emptyList();
+
+ List<Future<?>> futures = new ArrayList<>();
+
+ for (final AbstractCompactionTask task : tasks)
{
- protected void runMayThrow() throws IOException
+ Runnable runnable = new WrappedRunnable()
{
- if (tasks == null)
- return;
- for (AbstractCompactionTask task : tasks)
+ protected void runMayThrow() throws IOException
+ {
task.execute(metrics);
- }
- };
- return executor.submit(runnable);
+ }
+ };
+ futures.add(executor.submit(runnable));
+ }
+ return futures;
}
public void forceUserDefinedCompaction(String dataFiles)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c45ed171/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
index 912c7f1..c6e9445 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
@@ -20,7 +20,6 @@ package org.apache.cassandra.db.compaction;
import java.util.Collection;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
import org.apache.cassandra.db.*;
@@ -41,6 +40,7 @@ import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.Util.cellname;
import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
public class CompactionsPurgeTest extends SchemaLoader
@@ -85,7 +85,7 @@ public class CompactionsPurgeTest extends SchemaLoader
cfs.forceBlockingFlush();
// major compact and test that all columns but the resurrected one is completely gone
- CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE).get();
+ FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE));
cfs.invalidateCachedRow(key);
ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis()));
assertColumns(cf, "5");
@@ -346,8 +346,7 @@ public class CompactionsPurgeTest extends SchemaLoader
assertEquals(0, result.size());
// compact the two sstables with a gcBefore that does *not* allow the row tombstone to be purged
- Future<?> future = CompactionManager.instance.submitMaximal(cfs, (int) (System.currentTimeMillis() / 1000) - 10000);
- future.get();
+ FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, (int) (System.currentTimeMillis() / 1000) - 10000));
// the data should be gone, but the tombstone should still exist
assertEquals(1, cfs.getSSTables().size());
@@ -367,8 +366,7 @@ public class CompactionsPurgeTest extends SchemaLoader
cfs.forceBlockingFlush();
// compact the two sstables with a gcBefore that *does* allow the row tombstone to be purged
- future = CompactionManager.instance.submitMaximal(cfs, (int) (System.currentTimeMillis() / 1000) + 10000);
- future.get();
+ FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, (int) (System.currentTimeMillis() / 1000) + 10000));
// both the data and the tombstone should be gone this time
assertEquals(0, cfs.getSSTables().size());
[2/2] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/31099a16
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/31099a16
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/31099a16
Branch: refs/heads/trunk
Commit: 31099a1670e886eff744e31f8867124abb1cb7cb
Parents: 382834d c45ed17
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Dec 19 14:21:01 2014 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Fri Dec 19 14:21:01 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 27 ++++++++++++--------
.../db/compaction/CompactionsPurgeTest.java | 10 +++-----
3 files changed, 22 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/31099a16/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 689d36e,e5a8f05..5ec1bd2
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,47 -1,5 +1,48 @@@
+3.0
+ * Support index key/value entries on map collections (CASSANDRA-8473)
+ * Modernize schema tables (CASSANDRA-8261)
+ * Support for user-defined aggregation functions (CASSANDRA-8053)
+ * Fix NPE in SelectStatement with empty IN values (CASSANDRA-8419)
+ * Refactor SelectStatement, return IN results in natural order instead
+ of IN value list order (CASSANDRA-7981)
+ * Support UDTs, tuples, and collections in user-defined
+ functions (CASSANDRA-7563)
+ * Fix aggregate fn results on empty selection, result column name,
+ and cqlsh parsing (CASSANDRA-8229)
+ * Mark sstables as repaired after full repair (CASSANDRA-7586)
+ * Extend Descriptor to include a format value and refactor reader/writer apis (CASSANDRA-7443)
+ * Integrate JMH for microbenchmarks (CASSANDRA-8151)
+ * Keep sstable levels when bootstrapping (CASSANDRA-7460)
+ * Add Sigar library and perform basic OS settings check on startup (CASSANDRA-7838)
+ * Support for aggregation functions (CASSANDRA-4914)
+ * Remove cassandra-cli (CASSANDRA-7920)
+ * Accept dollar quoted strings in CQL (CASSANDRA-7769)
+ * Make assassinate a first class command (CASSANDRA-7935)
+ * Support IN clause on any clustering column (CASSANDRA-4762)
+ * Improve compaction logging (CASSANDRA-7818)
+ * Remove YamlFileNetworkTopologySnitch (CASSANDRA-7917)
+ * Do anticompaction in groups (CASSANDRA-6851)
+ * Support pure user-defined functions (CASSANDRA-7395, 7526, 7562, 7740, 7781, 7929,
+ 7924, 7812, 8063, 7813)
+ * Permit configurable timestamps with cassandra-stress (CASSANDRA-7416)
+ * Move sstable RandomAccessReader to nio2, which allows using the
+ FILE_SHARE_DELETE flag on Windows (CASSANDRA-4050)
+ * Remove CQL2 (CASSANDRA-5918)
+ * Add Thrift get_multi_slice call (CASSANDRA-6757)
+ * Optimize fetching multiple cells by name (CASSANDRA-6933)
+ * Allow compilation in java 8 (CASSANDRA-7028)
+ * Make incremental repair default (CASSANDRA-7250)
+ * Enable code coverage thru JaCoCo (CASSANDRA-7226)
+ * Switch external naming of 'column families' to 'tables' (CASSANDRA-4369)
+ * Shorten SSTable path (CASSANDRA-6962)
+ * Use unsafe mutations for most unit tests (CASSANDRA-6969)
+ * Fix race condition during calculation of pending ranges (CASSANDRA-7390)
+ * Fail on very large batch sizes (CASSANDRA-8011)
+ * Improve concurrency of repair (CASSANDRA-6455, 8208)
+
+
2.1.3
+ * Run major compactions for repaired/unrepaired in parallel (CASSANDRA-8510)
* (cqlsh) Fix compression options in DESCRIBE TABLE output when compression
is disabled (CASSANDRA-8288)
* (cqlsh) Fix DESCRIBE output after keyspaces are altered (CASSANDRA-7623)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/31099a16/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/31099a16/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
index 5f87ff3,c6e9445..4b5aa79
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
@@@ -20,17 -20,9 +20,16 @@@ package org.apache.cassandra.db.compact
import java.util.Collection;
import java.util.concurrent.ExecutionException;
- import java.util.concurrent.Future;
+import org.apache.cassandra.cache.CachingOptions;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.db.*;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.SimpleStrategy;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
@@@ -49,45 -40,13 +48,46 @@@ import static org.apache.cassandra.cql3
import static org.apache.cassandra.Util.cellname;
import org.apache.cassandra.utils.ByteBufferUtil;
+ import org.apache.cassandra.utils.FBUtilities;
-public class CompactionsPurgeTest extends SchemaLoader
+public class CompactionsPurgeTest
{
- public static final String KEYSPACE1 = "Keyspace1";
- public static final String KEYSPACE2 = "Keyspace2";
+ private static final String KEYSPACE1 = "CompactionsPurgeTest1";
+ private static final String CF_STANDARD1 = "Standard1";
+ private static final String CF_STANDARD2 = "Standard2";
+ private static final String KEYSPACE2 = "CompactionsPurgeTest2";
+ private static final String KEYSPACE_CACHED = "CompactionsPurgeTestCached";
+ private static final String CF_CACHED = "CachedCF";
+ private static final String KEYSPACE_CQL = "cql_keyspace";
+ private static final String CF_CQL = "table1";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2));
+ SchemaLoader.createKeyspace(KEYSPACE2,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE2, CF_STANDARD1));
+ SchemaLoader.createKeyspace(KEYSPACE_CACHED,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE_CACHED, CF_CACHED).caching(CachingOptions.ALL));
+ SchemaLoader.createKeyspace(KEYSPACE_CQL,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ CFMetaData.compile("CREATE TABLE " + CF_CQL + " ("
+ + "k int PRIMARY KEY,"
+ + "v1 text,"
+ + "v2 int"
+ + ")", KEYSPACE_CQL));
+ }
@Test
public void testMajorCompactionPurge() throws ExecutionException, InterruptedException