You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2019/08/07 09:05:49 UTC
[cassandra] branch cassandra-3.11 updated (b773bc7 -> 01c6daf)
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a change to branch cassandra-3.11
in repository https://gitbox.apache.org/repos/asf/cassandra.git.
from b773bc7 Make sure user defined compaction transactions are always closed
new 9af57a5 Filter sstables earlier when running cleanup
new 01c6daf Merge branch 'cassandra-3.0' into cassandra-3.11
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 1 +
.../cassandra/db/compaction/CompactionManager.java | 30 +++++++++---
test/unit/org/apache/cassandra/db/CleanupTest.java | 53 ++++++++++++++++++----
3 files changed, 68 insertions(+), 16 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-3.0' into cassandra-3.11
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch cassandra-3.11
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 01c6daf6e7e85df7e2c65e2db032df28c86aede0
Merge: b773bc7 9af57a5
Author: Marcus Eriksson <ma...@apache.org>
AuthorDate: Wed Aug 7 10:51:23 2019 +0200
Merge branch 'cassandra-3.0' into cassandra-3.11
CHANGES.txt | 1 +
.../cassandra/db/compaction/CompactionManager.java | 30 +++++++++---
test/unit/org/apache/cassandra/db/CleanupTest.java | 53 ++++++++++++++++++----
3 files changed, 68 insertions(+), 16 deletions(-)
diff --cc CHANGES.txt
index dc8baf2,a46a327..a7cae1d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,9 -1,5 +1,10 @@@
-3.0.19
+3.11.5
+ * Make sure user defined compaction transactions are always closed (CASSANDRA-15123)
+ * Fix cassandra-env.sh to use $CASSANDRA_CONF to find cassandra-jaas.config (CASSANDRA-14305)
+ * Fixed nodetool cfstats printing index name twice (CASSANDRA-14903)
+ * Add flag to disable SASI indexes, and warnings on creation (CASSANDRA-14866)
+Merged from 3.0:
+ * Filter sstables earlier when running cleanup (CASSANDRA-15100)
* Use mean row count instead of mean column count for index selectivity calculation (CASSANDRA-15259)
* Avoid updating unchanged gossip states (CASSANDRA-15097)
* Prevent recreation of previously dropped columns with a different kind (CASSANDRA-14948)
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 7086d77,694ad62..1a9da37
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -466,7 -465,25 +468,25 @@@ public class CompactionManager implemen
public Iterable<SSTableReader> filterSSTables(LifecycleTransaction transaction)
{
List<SSTableReader> sortedSSTables = Lists.newArrayList(transaction.originals());
- Collections.sort(sortedSSTables, SSTableReader.sizeComparator);
+ Iterator<SSTableReader> sstableIter = sortedSSTables.iterator();
+ int totalSSTables = 0;
+ int skippedSStables = 0;
+ while (sstableIter.hasNext())
+ {
+ SSTableReader sstable = sstableIter.next();
+ totalSSTables++;
+ if (!needsCleanup(sstable, ranges))
+ {
+ logger.debug("Not cleaning up {} ([{}, {}]) - no tokens outside owned ranges {}",
+ sstable, sstable.first.getToken(), sstable.last.getToken(), ranges);
+ sstableIter.remove();
+ transaction.cancel(sstable);
+ skippedSStables++;
+ }
+ }
+ logger.info("Skipping cleanup for {}/{} sstables for {}.{} since they are fully contained in owned ranges ({})",
+ skippedSStables, totalSSTables, cfStore.keyspace.getName(), cfStore.getTableName(), ranges);
- sortedSSTables.sort(new SSTableReader.SizeComparator());
++ sortedSSTables.sort(SSTableReader.sizeComparator);
return sortedSSTables;
}
diff --cc test/unit/org/apache/cassandra/db/CleanupTest.java
index 80e9b37,d4c613d..552e6d1
--- a/test/unit/org/apache/cassandra/db/CleanupTest.java
+++ b/test/unit/org/apache/cassandra/db/CleanupTest.java
@@@ -251,34 -235,42 +252,68 @@@ public class CleanupTes
assertTrue(cfs.getLiveSSTables().isEmpty());
}
+ @Test
+ public void testCleanupSkippingSSTables() throws UnknownHostException, ExecutionException, InterruptedException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE3);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD3);
+ cfs.disableAutoCompaction();
+ for (byte i = 0; i < 100; i++)
+ {
+ new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), ByteBuffer.wrap(new byte[] {i}))
+ .clustering(COLUMN)
+ .add("val", VALUE)
+ .build()
+ .applyUnsafe();
+ cfs.forceBlockingFlush();
+ }
+ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+ tmd.clearUnsafe();
+ tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.1"));
+ tmd.updateNormalToken(token(new byte[] {50}), InetAddress.getByName("127.0.0.1"));
+ Set<SSTableReader> beforeFirstCleanup = Sets.newHashSet(cfs.getLiveSSTables());
+ // single token - 127.0.0.1 owns everything, cleanup should be noop
+ cfs.forceCleanup(2);
+ assertEquals(beforeFirstCleanup, cfs.getLiveSSTables());
+ tmd.updateNormalToken(token(new byte[] {120}), InetAddress.getByName("127.0.0.2"));
+ cfs.forceCleanup(2);
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ assertEquals(sstable.first, sstable.last); // single-token sstables
+ assertTrue(sstable.first.getToken().compareTo(token(new byte[]{50})) <= 0);
+ // with single-token sstables they should all either be skipped or dropped:
+ assertTrue(beforeFirstCleanup.contains(sstable));
+ }
+ }
+
@Test
+ public void testuserDefinedCleanupWithNewToken() throws ExecutionException, InterruptedException, UnknownHostException
+ {
+ StorageService.instance.getTokenMetadata().clearUnsafe();
+
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, "val", LOOPS);
+
+ assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size());
+ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+
+ byte[] tk1 = new byte[1], tk2 = new byte[1];
+ tk1[0] = 2;
+ tk2[0] = 1;
+ tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
+ tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
+
+ for(SSTableReader r: cfs.getLiveSSTables())
+ CompactionManager.instance.forceUserDefinedCleanup(r.getFilename());
+
+ assertEquals(0, Util.getAll(Util.cmd(cfs).build()).size());
+ }
+
+ @Test
public void testNeedsCleanup() throws Exception
{
// setup
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org