You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2019/08/07 09:05:51 UTC
[cassandra] branch trunk updated (3ed1b79 -> af86247)
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git.
from 3ed1b79 Merge branch 'cassandra-3.11' into trunk
new 9af57a5 Filter sstables earlier when running cleanup
new 01c6daf Merge branch 'cassandra-3.0' into cassandra-3.11
new af86247 Merge branch 'cassandra-3.11' into trunk
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 1 +
.../cassandra/db/compaction/CompactionManager.java | 44 +++++++++++++++------
test/unit/org/apache/cassandra/db/CleanupTest.java | 45 ++++++++++++++++++++++
3 files changed, 78 insertions(+), 12 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-3.11' into trunk
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit af862475dff0945ff194eb041eab0a609cece0c1
Merge: 3ed1b79 01c6daf
Author: Marcus Eriksson <ma...@apache.org>
AuthorDate: Wed Aug 7 10:56:33 2019 +0200
Merge branch 'cassandra-3.11' into trunk
CHANGES.txt | 1 +
.../cassandra/db/compaction/CompactionManager.java | 44 +++++++++++++++------
test/unit/org/apache/cassandra/db/CleanupTest.java | 45 ++++++++++++++++++++++
3 files changed, 78 insertions(+), 12 deletions(-)
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 896fa2a,1a9da37..bb1d585
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -527,7 -468,25 +529,34 @@@ public class CompactionManager implemen
public Iterable<SSTableReader> filterSSTables(LifecycleTransaction transaction)
{
List<SSTableReader> sortedSSTables = Lists.newArrayList(transaction.originals());
- Collections.sort(sortedSSTables, SSTableReader.sizeComparator);
+ Iterator<SSTableReader> sstableIter = sortedSSTables.iterator();
+ int totalSSTables = 0;
+ int skippedSStables = 0;
+ while (sstableIter.hasNext())
+ {
+ SSTableReader sstable = sstableIter.next();
++ boolean needsCleanupFull = needsCleanup(sstable, fullRanges);
++ boolean needsCleanupTransient = needsCleanup(sstable, transientRanges);
++ //If there are no ranges for which the table needs cleanup either due to lack of intersection or lack
++ //of the table being repaired.
+ totalSSTables++;
- if (!needsCleanup(sstable, ranges))
++ if (!needsCleanupFull && (!needsCleanupTransient || !sstable.isRepaired()))
+ {
- logger.debug("Not cleaning up {} ([{}, {}]) - no tokens outside owned ranges {}",
- sstable, sstable.first.getToken(), sstable.last.getToken(), ranges);
++ logger.debug("Skipping {} ([{}, {}]) for cleanup; all rows should be kept. Needs cleanup full ranges: {} Needs cleanup transient ranges: {} Repaired: {}",
++ sstable,
++ sstable.first.getToken(),
++ sstable.last.getToken(),
++ needsCleanupFull,
++ needsCleanupTransient,
++ sstable.isRepaired());
+ sstableIter.remove();
+ transaction.cancel(sstable);
+ skippedSStables++;
+ }
+ }
- logger.info("Skipping cleanup for {}/{} sstables for {}.{} since they are fully contained in owned ranges ({})",
- skippedSStables, totalSSTables, cfStore.keyspace.getName(), cfStore.getTableName(), ranges);
++ logger.info("Skipping cleanup for {}/{} sstables for {}.{} since they are fully contained in owned ranges (full ranges: {}, transient ranges: {})",
++ skippedSStables, totalSSTables, cfStore.keyspace.getName(), cfStore.getTableName(), fullRanges, transientRanges);
+ sortedSSTables.sort(SSTableReader.sizeComparator);
return sortedSSTables;
}
@@@ -1174,16 -1139,7 +1203,7 @@@
{
txn.obsoleteOriginals();
txn.finish();
- return;
- }
-
- boolean needsCleanupFull = needsCleanup(sstable, fullRanges);
- boolean needsCleanupTransient = needsCleanup(sstable, transientRanges);
- //If there are no ranges for which the table needs cleanup either due to lack of intersection or lack
- //of the table being repaired.
- if (!needsCleanupFull && (!needsCleanupTransient || !sstable.isRepaired()))
- {
- logger.trace("Skipping {} for cleanup; all rows should be kept. Needs cleanup full ranges: {} Needs cleanup transient ranges: {} Repaired: {}", sstable, needsCleanupFull, needsCleanupTransient, sstable.isRepaired());
- logger.info("SSTable {} ([{}, {}]) does not intersect the owned ranges ({}), dropping it", sstable, sstable.first.getToken(), sstable.last.getToken(), ranges);
++ logger.info("SSTable {} ([{}, {}]) does not intersect the owned ranges ({}), dropping it", sstable, sstable.first.getToken(), sstable.last.getToken(), allRanges);
return;
}
diff --cc test/unit/org/apache/cassandra/db/CleanupTest.java
index 46c0afd,552e6d1..9965361
--- a/test/unit/org/apache/cassandra/db/CleanupTest.java
+++ b/test/unit/org/apache/cassandra/db/CleanupTest.java
@@@ -247,6 -252,40 +255,43 @@@ public class CleanupTes
assertTrue(cfs.getLiveSSTables().isEmpty());
}
+ @Test
+ public void testCleanupSkippingSSTables() throws UnknownHostException, ExecutionException, InterruptedException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE3);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD3);
+ cfs.disableAutoCompaction();
++ TokenMetadata tmd = StorageService.instance.getTokenMetadata();
++ tmd.clearUnsafe();
++ tmd.updateNormalToken(token(new byte[]{ 50 }), InetAddressAndPort.getByName("127.0.0.1"));
++
+ for (byte i = 0; i < 100; i++)
+ {
- new RowUpdateBuilder(cfs.metadata, System.currentTimeMillis(), ByteBuffer.wrap(new byte[] {i}))
- .clustering(COLUMN)
- .add("val", VALUE)
- .build()
- .applyUnsafe();
++ new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), ByteBuffer.wrap(new byte[]{ i }))
++ .clustering(COLUMN)
++ .add("val", VALUE)
++ .build()
++ .applyUnsafe();
+ cfs.forceBlockingFlush();
+ }
- TokenMetadata tmd = StorageService.instance.getTokenMetadata();
- tmd.clearUnsafe();
- tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.1"));
- tmd.updateNormalToken(token(new byte[] {50}), InetAddress.getByName("127.0.0.1"));
++
+ Set<SSTableReader> beforeFirstCleanup = Sets.newHashSet(cfs.getLiveSSTables());
+ // single token - 127.0.0.1 owns everything, cleanup should be noop
+ cfs.forceCleanup(2);
+ assertEquals(beforeFirstCleanup, cfs.getLiveSSTables());
- tmd.updateNormalToken(token(new byte[] {120}), InetAddress.getByName("127.0.0.2"));
++ tmd.updateNormalToken(token(new byte[]{ 120 }), InetAddressAndPort.getByName("127.0.0.2"));
++
+ cfs.forceCleanup(2);
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ assertEquals(sstable.first, sstable.last); // single-token sstables
- assertTrue(sstable.first.getToken().compareTo(token(new byte[]{50})) <= 0);
++ assertTrue(sstable.first.getToken().compareTo(token(new byte[]{ 50 })) <= 0);
+ // with single-token sstables they should all either be skipped or dropped:
+ assertTrue(beforeFirstCleanup.contains(sstable));
+ }
++
+ }
+
@Test
public void testuserDefinedCleanupWithNewToken() throws ExecutionException, InterruptedException, UnknownHostException
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org