You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2014/02/13 08:48:04 UTC
[1/2] git commit: Avoid overlaps in LCS by not skipping levels and
verifying that the candidates don't overlap any compacting sstables.
Updated Branches:
refs/heads/trunk 4334f99f4 -> 3aaa0295c
Avoid overlaps in LCS by not skipping levels and verifying that the candidates
don't overlap any compacting sstables.
Patch by marcuse, reviewed by jbellis for CASSANDRA-6688
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/15ee9489
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/15ee9489
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/15ee9489
Branch: refs/heads/trunk
Commit: 15ee94897d7d8dae5b23c2a2112eb1564fada22b
Parents: 5f60fcc
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Feb 13 08:42:13 2014 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 13 08:42:13 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../db/compaction/LeveledManifest.java | 23 ++++----------------
2 files changed, 6 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15ee9489/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a4dc8fd..9509a76 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,4 @@
+2.0.6
* Add compatibility for Hadoop 0.2.x (CASSANDRA-5201)
* Fix EstimatedHistogram races (CASSANDRA-6682)
* Failure detector correctly converts initial value to nanos (CASSANDRA-6658)
@@ -10,6 +11,7 @@
time histogram (CASSANDRA-6522)
* Stop CommitLogSegment.close() from calling sync() (CASSANDRA-6652)
* Make commitlog failure handling configurable (CASSANDRA-6364)
+ * Avoid overlaps in LCS (CASSANDRA-6688)
Merged from 1.2:
* Fix broken streams when replacing with same IP (CASSANDRA-6622)
* Fix upgradesstables NPE for non-CF-based indexes (CASSANDRA-6645)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15ee9489/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 4347ad5..a78a867 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -113,23 +113,6 @@ public class LeveledManifest
generations[level].add(reader);
}
- /**
- * if the number of SSTables in the current compacted set *by itself* exceeds the target level's
- * (regardless of the level's current contents), find an empty level instead
- */
- private int skipLevels(int newLevel, Iterable<SSTableReader> added)
- {
- // Note that we now check if the sstables included in the compaction, *before* the compaction, fit in the next level.
- // This is needed since we need to decide before the actual compaction what level they will be in.
- // This should be safe, we might skip levels where the compacted data could have fit but that should be ok.
- while (maxBytesForLevel(newLevel) < SSTableReader.getTotalBytes(added)
- && generations[(newLevel + 1)].isEmpty())
- {
- newLevel++;
- }
- return newLevel;
- }
-
public synchronized void replace(Collection<SSTableReader> removed, Collection<SSTableReader> added)
{
assert !removed.isEmpty(); // use add() instead of promote when adding new sstables
@@ -449,7 +432,10 @@ public class LeveledManifest
for (SSTableReader newCandidate : overlappedL0)
{
- candidates.add(newCandidate);
+ // overlappedL0 could contain sstables that are not in compactingL0, but do overlap
+ // other sstables that are
+ if (overlapping(newCandidate, compactingL0).isEmpty())
+ candidates.add(newCandidate);
remaining.remove(newCandidate);
}
@@ -572,7 +558,6 @@ public class LeveledManifest
else
{
newLevel = minimumLevel == maximumLevel ? maximumLevel + 1 : maximumLevel;
- newLevel = skipLevels(newLevel, sstables);
assert newLevel > 0;
}
return newLevel;
[2/2] git commit: Merge branch 'cassandra-2.0' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-2.0' into trunk
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3aaa0295
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3aaa0295
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3aaa0295
Branch: refs/heads/trunk
Commit: 3aaa0295c884b3c8d8e6ab5e3cd90e26cc77d1a3
Parents: 4334f99 15ee948
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Feb 13 08:46:59 2014 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 13 08:46:59 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 4 +++-
.../db/compaction/LeveledManifest.java | 23 ++++----------------
2 files changed, 7 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3aaa0295/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a45df89,9509a76..139eb06
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,37 -1,3 +1,36 @@@
+2.1
+ * add listsnapshots command to nodetool (CASSANDRA-5742)
+ * Introduce AtomicBTreeColumns (CASSANDRA-6271)
+ * Multithreaded commitlog (CASSANDRA-3578)
+ * allocate fixed index summary memory pool and resample cold index summaries
+ to use less memory (CASSANDRA-5519)
+ * Removed multithreaded compaction (CASSANDRA-6142)
+ * Parallelize fetching rows for low-cardinality indexes (CASSANDRA-1337)
+ * change logging from log4j to logback (CASSANDRA-5883)
+ * switch to LZ4 compression for internode communication (CASSANDRA-5887)
+ * Stop using Thrift-generated Index* classes internally (CASSANDRA-5971)
+ * Remove 1.2 network compatibility code (CASSANDRA-5960)
+ * Remove leveled json manifest migration code (CASSANDRA-5996)
+ * Remove CFDefinition (CASSANDRA-6253)
+ * Use AtomicIntegerFieldUpdater in RefCountedMemory (CASSANDRA-6278)
+ * User-defined types for CQL3 (CASSANDRA-5590)
+ * Use of o.a.c.metrics in nodetool (CASSANDRA-5871, 6406)
+ * Batch read from OTC's queue and cleanup (CASSANDRA-1632)
+ * Secondary index support for collections (CASSANDRA-4511, 6383)
+ * SSTable metadata(Stats.db) format change (CASSANDRA-6356)
+ * Push composites support in the storage engine
+ (CASSANDRA-5417, CASSANDRA-6520)
+ * Add snapshot space used to cfstats (CASSANDRA-6231)
+ * Add cardinality estimator for key count estimation (CASSANDRA-5906)
+ * CF id is changed to be non-deterministic. Data dir/key cache are created
+ uniquely for CF id (CASSANDRA-5202)
+ * New counters implementation (CASSANDRA-6504)
+ * Replace UnsortedColumns, EmptyColumns, TreeMapBackedSortedColumns with new
+ ArrayBackedSortedColumns (CASSANDRA-6630, CASSANDRA-6662, CASSANDRA-6690)
+ * Add option to use row cache with a given amount of rows (CASSANDRA-5357)
+ * Avoid repairing already repaired data (CASSANDRA-5351)
+ * Reject counter updates with USING TTL/TIMESTAMP (CASSANDRA-6649)
+
-
2.0.6
* Add compatibility for Hadoop 0.2.x (CASSANDRA-5201)
* Fix EstimatedHistogram races (CASSANDRA-6682)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3aaa0295/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index cab726d,a78a867..05b838d
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@@ -103,93 -105,14 +103,76 @@@ public class LeveledManifes
public synchronized void add(SSTableReader reader)
{
+ if (!hasRepairedData && reader.isRepaired())
+ {
+ // this is the first repaired sstable we get - we need to
+ // rebuild the entire manifest, unrepaired data should be
+ // in unrepairedL0. Note that we keep the sstable level in
+ // the sstable metadata since we are likely to be able to
+ // re-add it at a good level later (during anticompaction
+ // for example).
+ hasRepairedData = true;
+ rebuildManifestAfterFirstRepair();
+ }
+
int level = reader.getSSTableLevel();
- assert level < generations.length : "Invalid level " + level + " out of " + (generations.length - 1);
- logDistribution();
+ if (hasRepairedData && !reader.isRepaired())
+ {
+ logger.debug("Adding unrepaired {} to unrepaired L0", reader);
+ unrepairedL0.add(reader);
+ }
+ else
+ {
+ assert level < generations.length : "Invalid level " + level + " out of " + (generations.length - 1);
+ logDistribution();
+ if (canAddSSTable(reader))
+ {
+ // adding the sstable does not cause overlap in the level
+ logger.debug("Adding {} to L{}", reader, level);
+ generations[level].add(reader);
+ }
+ else
+ {
+ // this can happen if:
+ // * a compaction has promoted an overlapping sstable to the given level, or
+ // * we promote a non-repaired sstable to repaired at level > 0, but an ongoing compaction
+ // was also supposed to add an sstable at the given level.
+ //
+ // The add(..):ed sstable will be sent to level 0
+ try
+ {
+ reader.descriptor.getMetadataSerializer().mutateLevel(reader.descriptor, 0);
+ reader.reloadSSTableMetadata();
+ }
+ catch (IOException e)
+ {
+ logger.error("Could not change sstable level - adding it at level 0 anyway, we will find it at restart.", e);
+ }
+ generations[0].add(reader);
+ }
+ }
- logger.debug("Adding {} to L{}", reader, level);
- generations[level].add(reader);
+ }
+
+
+ /**
+ * Since we run standard LCS when we have no repaired data
+ * we need to move all sstables from the leveling
+ * to unrepairedL0.
+ */
+ private void rebuildManifestAfterFirstRepair()
+ {
+ for (int i = 1; i < getAllLevelSize().length; i++)
+ {
+
+ for (SSTableReader sstable : getLevel(i))
+ {
+ generations[i] = new ArrayList<>();
+ add(sstable);
+ }
+ }
}
- /**
- * if the number of SSTables in the current compacted set *by itself* exceeds the target level's
- * (regardless of the level's current contents), find an empty level instead
- */
- private int skipLevels(int newLevel, Iterable<SSTableReader> added)
- {
- // Note that we now check if the sstables included in the compaction, *before* the compaction, fit in the next level.
- // This is needed since we need to decide before the actual compaction what level they will be in.
- // This should be safe, we might skip levels where the compacted data could have fit but that should be ok.
- while (maxBytesForLevel(newLevel) < SSTableReader.getTotalBytes(added)
- && getLevel(newLevel + 1).isEmpty())
- {
- newLevel++;
- }
- return newLevel;
- }
-
public synchronized void replace(Collection<SSTableReader> removed, Collection<SSTableReader> added)
{
assert !removed.isEmpty(); // use add() instead of promote when adding new sstables