You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2013/08/29 19:53:07 UTC
[1/3] git commit: Improve leveled compaction's ability to find
non-overlapping L0 compactions to work on concurrently patch by marcuse and
jbellis; reviewed by yukim for CASSANDRA-5921
Updated Branches:
refs/heads/cassandra-2.0 9495eb59c -> c1e0f3102
refs/heads/trunk c146f437b -> 73e27b07f
Improve leveled compaction's ability to find non-overlapping L0 compactions to work on concurrently
patch by marcuse and jbellis; reviewed by yukim for CASSANDRA-5921
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c1e0f310
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c1e0f310
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c1e0f310
Branch: refs/heads/cassandra-2.0
Commit: c1e0f3102ebd5670927ce89420681971b8325379
Parents: 9495eb5
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Aug 29 12:51:58 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Aug 29 12:51:58 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../db/compaction/LeveledManifest.java | 20 +++++++++++---------
2 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c1e0f310/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8a1004f..2d3ee24 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,6 @@
2.0.1
+ * Improve leveled compaction's ability to find non-overlapping L0 compactions
+ to work on concurrently (CASSANDRA-5921)
* Notify indexer of columns shadowed by range tombstones (CASSANDRA-5614)
* Log Merkle tree stats (CASSANDRA-2698)
* Switch from crc32 to adler32 for compressed sstable checksums (CASSANDRA-5862)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c1e0f310/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 597b851..bc6824a 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -25,6 +25,7 @@ import java.util.*;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -412,6 +413,8 @@ public class LeveledManifest
if (level == 0)
{
+ Set<SSTableReader> compactingL0 = ImmutableSet.copyOf(Iterables.filter(generations[0], Predicates.in(compacting)));
+
// L0 is the dumping ground for new sstables which thus may overlap each other.
//
// We treat L0 compactions specially:
@@ -433,13 +436,14 @@ public class LeveledManifest
if (candidates.contains(sstable))
continue;
- for (SSTableReader newCandidate : Sets.union(Collections.singleton(sstable), overlapping(sstable, remaining)))
+ Sets.SetView<SSTableReader> overlappedL0 = Sets.union(Collections.singleton(sstable), overlapping(sstable, remaining));
+ if (!Sets.intersection(overlappedL0, compactingL0).isEmpty())
+ continue;
+
+ for (SSTableReader newCandidate : overlappedL0)
{
- if (!compacting.contains(newCandidate))
- {
- candidates.add(newCandidate);
- remaining.remove(newCandidate);
- }
+ candidates.add(newCandidate);
+ remaining.remove(newCandidate);
}
if (candidates.size() > MAX_COMPACTING_L0)
@@ -458,9 +462,7 @@ public class LeveledManifest
// TODO try to find a set of L0 sstables that only overlaps with non-busy L1 sstables
candidates = Sets.union(candidates, overlapping(candidates, generations[1]));
}
- // check overlap with L0 compacting sstables to make sure we are not generating overlap in L1.
- Iterable<SSTableReader> compactingL0 = Iterables.filter(generations[0], Predicates.in(compacting));
- if (candidates.size() < 2 || !Sets.intersection(candidates, compacting).isEmpty() || !overlapping(candidates, compactingL0).isEmpty())
+ if (candidates.size() < 2)
return Collections.emptyList();
else
return candidates;
[2/3] git commit: Improve leveled compaction's ability to find
non-overlapping L0 compactions to work on concurrently patch by marcuse and
jbellis; reviewed by yukim for CASSANDRA-5921
Posted by jb...@apache.org.
Improve leveled compaction's ability to find non-overlapping L0 compactions to work on concurrently
patch by marcuse and jbellis; reviewed by yukim for CASSANDRA-5921
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c1e0f310
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c1e0f310
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c1e0f310
Branch: refs/heads/trunk
Commit: c1e0f3102ebd5670927ce89420681971b8325379
Parents: 9495eb5
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Aug 29 12:51:58 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Aug 29 12:51:58 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../db/compaction/LeveledManifest.java | 20 +++++++++++---------
2 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c1e0f310/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8a1004f..2d3ee24 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,6 @@
2.0.1
+ * Improve leveled compaction's ability to find non-overlapping L0 compactions
+ to work on concurrently (CASSANDRA-5921)
* Notify indexer of columns shadowed by range tombstones (CASSANDRA-5614)
* Log Merkle tree stats (CASSANDRA-2698)
* Switch from crc32 to adler32 for compressed sstable checksums (CASSANDRA-5862)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c1e0f310/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 597b851..bc6824a 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -25,6 +25,7 @@ import java.util.*;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -412,6 +413,8 @@ public class LeveledManifest
if (level == 0)
{
+ Set<SSTableReader> compactingL0 = ImmutableSet.copyOf(Iterables.filter(generations[0], Predicates.in(compacting)));
+
// L0 is the dumping ground for new sstables which thus may overlap each other.
//
// We treat L0 compactions specially:
@@ -433,13 +436,14 @@ public class LeveledManifest
if (candidates.contains(sstable))
continue;
- for (SSTableReader newCandidate : Sets.union(Collections.singleton(sstable), overlapping(sstable, remaining)))
+ Sets.SetView<SSTableReader> overlappedL0 = Sets.union(Collections.singleton(sstable), overlapping(sstable, remaining));
+ if (!Sets.intersection(overlappedL0, compactingL0).isEmpty())
+ continue;
+
+ for (SSTableReader newCandidate : overlappedL0)
{
- if (!compacting.contains(newCandidate))
- {
- candidates.add(newCandidate);
- remaining.remove(newCandidate);
- }
+ candidates.add(newCandidate);
+ remaining.remove(newCandidate);
}
if (candidates.size() > MAX_COMPACTING_L0)
@@ -458,9 +462,7 @@ public class LeveledManifest
// TODO try to find a set of L0 sstables that only overlaps with non-busy L1 sstables
candidates = Sets.union(candidates, overlapping(candidates, generations[1]));
}
- // check overlap with L0 compacting sstables to make sure we are not generating overlap in L1.
- Iterable<SSTableReader> compactingL0 = Iterables.filter(generations[0], Predicates.in(compacting));
- if (candidates.size() < 2 || !Sets.intersection(candidates, compacting).isEmpty() || !overlapping(candidates, compactingL0).isEmpty())
+ if (candidates.size() < 2)
return Collections.emptyList();
else
return candidates;
[3/3] git commit: merge from 2.0
Posted by jb...@apache.org.
merge from 2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/73e27b07
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/73e27b07
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/73e27b07
Branch: refs/heads/trunk
Commit: 73e27b07f588b0ace8255090d3840903893a7435
Parents: c146f43 c1e0f31
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Aug 29 12:52:58 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Aug 29 12:52:58 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../db/compaction/LeveledManifest.java | 20 +++++++++++---------
2 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73e27b07/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fdfcc0a,2d3ee24..7d7feaf
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,8 -1,6 +1,10 @@@
+2.1
+ * change logging from log4j to logback (CASSANDRA-5883)
+
+
2.0.1
+ * Improve leveled compaction's ability to find non-overlapping L0 compactions
+ to work on concurrently (CASSANDRA-5921)
* Notify indexer of columns shadowed by range tombstones (CASSANDRA-5614)
* Log Merkle tree stats (CASSANDRA-2698)
* Switch from crc32 to adler32 for compressed sstable checksums (CASSANDRA-5862)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/73e27b07/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------