You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2016/08/23 14:50:28 UTC
[1/5] cassandra git commit: Calculate last compacted keys from higher
level
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 d0fc95312 -> 8bd629942
refs/heads/trunk 5dda0abbf -> bdeef40d4
Calculate last compacted keys from higher level
patch by Dikang Gu; reviewed by marcuse for CASSANDRA-6216
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6808e755
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6808e755
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6808e755
Branch: refs/heads/cassandra-3.0
Commit: 6808e7555762236b6222f65bf066f90f7597405e
Parents: d0fc953
Author: Dikang Gu <di...@gmail.com>
Authored: Sat Aug 20 00:55:22 2016 -0700
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 23 16:18:39 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../compaction/LeveledCompactionStrategy.java | 7 ++++
.../db/compaction/LeveledManifest.java | 36 ++++++++++++++++++++
3 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d384fb3..0d461f3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.9
+ * Calculate last compacted key on startup (CASSANDRA-6216)
* Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190)
* Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
* Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index cd74620..65fca27 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@ -85,6 +85,13 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
return manifest.getAllLevelSize();
}
+ @Override
+ public void startup()
+ {
+ manifest.calculateLastCompactedKeys();
+ super.startup();
+ }
+
/**
* the only difference between background and maximal in LCS is that maximal is still allowed
* (by explicit user request) even when compaction is disabled.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 2bfe88f..cf3ae1a 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -30,6 +30,7 @@ import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,6 +67,7 @@ public class LeveledManifest
@VisibleForTesting
protected final List<SSTableReader>[] generations;
private final PartitionPosition[] lastCompactedKeys;
+ private volatile boolean calculatedLastCompactedKeys = false;
private final long maxSSTableSizeInBytes;
private final SizeTieredCompactionStrategyOptions options;
private final int [] compactionCounter;
@@ -104,9 +106,38 @@ public class LeveledManifest
{
manifest.repairOverlappingSSTables(i);
}
+ manifest.calculateLastCompactedKeys();
return manifest;
}
+ /**
+ * If we want to start compaction in level n, find the newest (by modification time) file in level n+1
+ * and use its last token for last compacted key in level n;
+ */
+ public void calculateLastCompactedKeys()
+ {
+ for (int i = 0; i < generations.length - 1; i++)
+ {
+ // this level is empty
+ if (generations[i + 1].isEmpty())
+ continue;
+
+ SSTableReader sstableWithMaxModificationTime = null;
+ long maxModificationTime = Long.MIN_VALUE;
+ for (SSTableReader ssTableReader : generations[i + 1])
+ {
+ long modificationTime = ssTableReader.getCreationTimeFor(Component.DATA);
+ if (modificationTime >= maxModificationTime)
+ {
+ sstableWithMaxModificationTime = ssTableReader;
+ maxModificationTime = modificationTime;
+ }
+ }
+
+ lastCompactedKeys[i] = sstableWithMaxModificationTime.last;
+ }
+ }
+
public synchronized void add(SSTableReader reader)
{
int level = reader.getSSTableLevel();
@@ -307,6 +338,11 @@ public class LeveledManifest
// This isn't a magic wand -- if you are consistently writing too fast for LCS to keep
// up, you're still screwed. But if instead you have intermittent bursts of activity,
// it can help a lot.
+ if (!calculatedLastCompactedKeys)
+ {
+ this.calculateLastCompactedKeys();
+ }
+
for (int i = generations.length - 1; i > 0; i--)
{
List<SSTableReader> sstables = getLevel(i);
[4/5] cassandra git commit: Disk failure policy should not be invoked
on out of space
Posted by ma...@apache.org.
Disk failure policy should not be invoked on out of space
patch by Sankalp Kohli; reviewed by marcuse for CASSANDRA-12385
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8bd62994
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8bd62994
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8bd62994
Branch: refs/heads/cassandra-3.0
Commit: 8bd629942ca7014ff5b6e001db10290691f6cc08
Parents: 6808e75
Author: sankalp kohli <sa...@apple.com>
Authored: Fri Aug 19 09:44:48 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 23 16:32:11 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/Directories.java | 3 +-
.../db/compaction/AbstractCompactionTask.java | 7 +++++
.../cassandra/io/FSDiskFullWriteError.java | 33 ++++++++++++++++++++
4 files changed, 43 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8bd62994/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0d461f3..bd1fc5d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.9
+ * Disk failure policy should not be invoked on out of space (CASSANDRA-12385)
* Calculate last compacted key on startup (CASSANDRA-6216)
* Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190)
* Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8bd62994/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 30fe56d..68aa6be 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -47,6 +47,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.FSDiskFullWriteError;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.FileUtils;
@@ -385,7 +386,7 @@ public class Directories
if (candidates.isEmpty())
if (tooBig)
- throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
+ throw new FSDiskFullWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
else
throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8bd62994/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
index 155bf2f..430c916 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
@@ -23,6 +23,7 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorStatsCollector;
import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.io.FSDiskFullWriteError;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
@@ -59,6 +60,12 @@ public abstract class AbstractCompactionTask extends WrappedRunnable
{
return executeInternal(collector);
}
+ catch(FSDiskFullWriteError e)
+ {
+ RuntimeException cause = new RuntimeException("Converted from FSDiskFullWriteError: " + e.getMessage());
+ cause.setStackTrace(e.getStackTrace());
+ throw new RuntimeException("Throwing new Runtime to bypass exception handler when disk is full", cause);
+ }
finally
{
transaction.close();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8bd62994/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java b/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java
new file mode 100644
index 0000000..ca5d8da
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io;
+
+public class FSDiskFullWriteError extends FSWriteError
+{
+ public FSDiskFullWriteError(Throwable cause, String path)
+ {
+ super(cause, path);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "FSDiskFullWriteError in " + path;
+ }
+}
[5/5] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bdeef40d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bdeef40d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bdeef40d
Branch: refs/heads/trunk
Commit: bdeef40d4776085c8194d144875f2e15be7955cf
Parents: 5dda0ab 8bd6299
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Aug 23 16:46:16 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 23 16:46:16 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../org/apache/cassandra/db/Directories.java | 3 +-
.../db/compaction/AbstractCompactionTask.java | 7 ++++
.../compaction/LeveledCompactionStrategy.java | 7 ++++
.../db/compaction/LeveledManifest.java | 36 ++++++++++++++++++++
.../cassandra/io/FSDiskFullWriteError.java | 33 ++++++++++++++++++
6 files changed, 87 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bdeef40d/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 8a95b98,bd1fc5d..fd843dd
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,94 -1,13 +1,96 @@@
-3.0.9
+3.10
+ * "SELECT * FROM foo LIMIT ;" does not error out (CASSANDRA-12154)
+ * Define executeLocally() at the ReadQuery Level (CASSANDRA-12474)
+ * Extend read/write failure messages with a map of replica addresses
+ to error codes in the v5 native protocol (CASSANDRA-12311)
+ * Fix rebuild of SASI indexes with existing index files (CASSANDRA-12374)
+ * Let DatabaseDescriptor not implicitly startup services (CASSANDRA-9054)
+ * Fix clustering indexes in presence of static columns in SASI (CASSANDRA-12378)
+ * Fix queries on columns with reversed type on SASI indexes (CASSANDRA-12223)
+ * Added slow query log (CASSANDRA-12403)
+ * Count full coordinated request against timeout (CASSANDRA-12256)
+ * Allow TTL with null value on insert and update (CASSANDRA-12216)
+ * Make decommission operation resumable (CASSANDRA-12008)
+ * Add support to one-way targeted repair (CASSANDRA-9876)
+ * Remove clientutil jar (CASSANDRA-11635)
+ * Fix compaction throughput throttle (CASSANDRA-12366)
+ * Delay releasing Memtable memory on flush until PostFlush has finished running (CASSANDRA-12358)
+ * Cassandra stress should dump all setting on startup (CASSANDRA-11914)
+ * Make it possible to compact a given token range (CASSANDRA-10643)
+ * Allow updating DynamicEndpointSnitch properties via JMX (CASSANDRA-12179)
+ * Collect metrics on queries by consistency level (CASSANDRA-7384)
+ * Add support for GROUP BY to SELECT statement (CASSANDRA-10707)
+ * Deprecate memtable_cleanup_threshold and update default for memtable_flush_writers (CASSANDRA-12228)
+ * Upgrade to OHC 0.4.4 (CASSANDRA-12133)
+ * Add version command to cassandra-stress (CASSANDRA-12258)
+ * Create compaction-stress tool (CASSANDRA-11844)
+ * Garbage-collecting compaction operation and schema option (CASSANDRA-7019)
+ * Add beta protocol flag for v5 native protocol (CASSANDRA-12142)
+ * Support filtering on non-PRIMARY KEY columns in the CREATE
+ MATERIALIZED VIEW statement's WHERE clause (CASSANDRA-10368)
+ * Unify STDOUT and SYSTEMLOG logback format (CASSANDRA-12004)
+ * COPY FROM should raise error for non-existing input files (CASSANDRA-12174)
+ * Faster write path (CASSANDRA-12269)
+ * Option to leave omitted columns in INSERT JSON unset (CASSANDRA-11424)
+ * Support json/yaml output in nodetool tpstats (CASSANDRA-12035)
+ * Expose metrics for successful/failed authentication attempts (CASSANDRA-10635)
+ * Prepend snapshot name with "truncated" or "dropped" when a snapshot
+ is taken before truncating or dropping a table (CASSANDRA-12178)
+ * Optimize RestrictionSet (CASSANDRA-12153)
+ * cqlsh does not automatically downgrade CQL version (CASSANDRA-12150)
+ * Omit (de)serialization of state variable in UDAs (CASSANDRA-9613)
+ * Create a system table to expose prepared statements (CASSANDRA-8831)
+ * Reuse DataOutputBuffer from ColumnIndex (CASSANDRA-11970)
+ * Remove DatabaseDescriptor dependency from SegmentedFile (CASSANDRA-11580)
+ * Add supplied username to authentication error messages (CASSANDRA-12076)
+ * Remove pre-startup check for open JMX port (CASSANDRA-12074)
+ * Remove compaction Severity from DynamicEndpointSnitch (CASSANDRA-11738)
+ * Restore resumable hints delivery (CASSANDRA-11960)
+Merged from 3.0:
+ * Disk failure policy should not be invoked on out of space (CASSANDRA-12385)
+ * Calculate last compacted key on startup (CASSANDRA-6216)
* Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190)
+
+
+3.9
+ * Fix value skipping with counter columns (CASSANDRA-11726)
+ * Fix nodetool tablestats miss SSTable count (CASSANDRA-12205)
+ * Fixed flacky SSTablesIteratedTest (CASSANDRA-12282)
+ * Fixed flacky SSTableRewriterTest: check file counts before calling validateCFS (CASSANDRA-12348)
+ * cqlsh: Fix handling of $$-escaped strings (CASSANDRA-12189)
+ * Fix SSL JMX requiring truststore containing server cert (CASSANDRA-12109)
+ * RTE from new CDC column breaks in flight queries (CASSANDRA-12236)
+ * Fix hdr logging for single operation workloads (CASSANDRA-12145)
+ * Fix SASI PREFIX search in CONTAINS mode with partial terms (CASSANDRA-12073)
+ * Increase size of flushExecutor thread pool (CASSANDRA-12071)
+ * Partial revert of CASSANDRA-11971, cannot recycle buffer in SP.sendMessagesToNonlocalDC (CASSANDRA-11950)
+ * Upgrade netty to 4.0.39 (CASSANDRA-12032, CASSANDRA-12034)
+ * Improve details in compaction log message (CASSANDRA-12080)
+ * Allow unset values in CQLSSTableWriter (CASSANDRA-11911)
+ * Chunk cache to request compressor-compatible buffers if pool space is exhausted (CASSANDRA-11993)
+ * Remove DatabaseDescriptor dependencies from SequentialWriter (CASSANDRA-11579)
+ * Move skip_stop_words filter before stemming (CASSANDRA-12078)
+ * Support seek() in EncryptedFileSegmentInputStream (CASSANDRA-11957)
+ * SSTable tools mishandling LocalPartitioner (CASSANDRA-12002)
+ * When SEPWorker assigned work, set thread name to match pool (CASSANDRA-11966)
+ * Add cross-DC latency metrics (CASSANDRA-11596)
+ * Allow terms in selection clause (CASSANDRA-10783)
+ * Add bind variables to trace (CASSANDRA-11719)
+ * Switch counter shards' clock to timestamps (CASSANDRA-9811)
+ * Introduce HdrHistogram and response/service/wait separation to stress tool (CASSANDRA-11853)
+ * entry-weighers in QueryProcessor should respect partitionKeyBindIndexes field (CASSANDRA-11718)
+ * Support older ant versions (CASSANDRA-11807)
+ * Estimate compressed on disk size when deciding if sstable size limit reached (CASSANDRA-11623)
+ * cassandra-stress profiles should support case sensitive schemas (CASSANDRA-11546)
+ * Remove DatabaseDescriptor dependency from FileUtils (CASSANDRA-11578)
+ * Faster streaming (CASSANDRA-9766)
+ * Add prepared query parameter to trace for "Execute CQL3 prepared query" session (CASSANDRA-11425)
+ * Add repaired percentage metric (CASSANDRA-11503)
+ * Add Change-Data-Capture (CASSANDRA-8844)
+Merged from 3.0:
* Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
* Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331)
- * Backport CASSANDRA-12002 (CASSANDRA-12177)
* Make sure compaction stats are updated when compaction is interrupted (CASSANDRA-12100)
- * Fix potential bad messaging service message for paged range reads
- within mixed-version 3.x clusters (CASSANDRA-12249)
* Change commitlog and sstables to track dirty and clean intervals (CASSANDRA-11828)
* NullPointerException during compaction on table with static columns (CASSANDRA-12336)
* Fixed ConcurrentModificationException when reading metrics in GraphiteReporter (CASSANDRA-11823)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bdeef40d/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bdeef40d/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bdeef40d/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
[2/5] cassandra git commit: Calculate last compacted keys from higher
level
Posted by ma...@apache.org.
Calculate last compacted keys from higher level
patch by Dikang Gu; reviewed by marcuse for CASSANDRA-6216
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6808e755
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6808e755
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6808e755
Branch: refs/heads/trunk
Commit: 6808e7555762236b6222f65bf066f90f7597405e
Parents: d0fc953
Author: Dikang Gu <di...@gmail.com>
Authored: Sat Aug 20 00:55:22 2016 -0700
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 23 16:18:39 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../compaction/LeveledCompactionStrategy.java | 7 ++++
.../db/compaction/LeveledManifest.java | 36 ++++++++++++++++++++
3 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d384fb3..0d461f3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.9
+ * Calculate last compacted key on startup (CASSANDRA-6216)
* Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190)
* Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
* Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index cd74620..65fca27 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@ -85,6 +85,13 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
return manifest.getAllLevelSize();
}
+ @Override
+ public void startup()
+ {
+ manifest.calculateLastCompactedKeys();
+ super.startup();
+ }
+
/**
* the only difference between background and maximal in LCS is that maximal is still allowed
* (by explicit user request) even when compaction is disabled.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 2bfe88f..cf3ae1a 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -30,6 +30,7 @@ import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,6 +67,7 @@ public class LeveledManifest
@VisibleForTesting
protected final List<SSTableReader>[] generations;
private final PartitionPosition[] lastCompactedKeys;
+ private volatile boolean calculatedLastCompactedKeys = false;
private final long maxSSTableSizeInBytes;
private final SizeTieredCompactionStrategyOptions options;
private final int [] compactionCounter;
@@ -104,9 +106,38 @@ public class LeveledManifest
{
manifest.repairOverlappingSSTables(i);
}
+ manifest.calculateLastCompactedKeys();
return manifest;
}
+ /**
+ * If we want to start compaction in level n, find the newest (by modification time) file in level n+1
+ * and use its last token for last compacted key in level n;
+ */
+ public void calculateLastCompactedKeys()
+ {
+ for (int i = 0; i < generations.length - 1; i++)
+ {
+ // this level is empty
+ if (generations[i + 1].isEmpty())
+ continue;
+
+ SSTableReader sstableWithMaxModificationTime = null;
+ long maxModificationTime = Long.MIN_VALUE;
+ for (SSTableReader ssTableReader : generations[i + 1])
+ {
+ long modificationTime = ssTableReader.getCreationTimeFor(Component.DATA);
+ if (modificationTime >= maxModificationTime)
+ {
+ sstableWithMaxModificationTime = ssTableReader;
+ maxModificationTime = modificationTime;
+ }
+ }
+
+ lastCompactedKeys[i] = sstableWithMaxModificationTime.last;
+ }
+ }
+
public synchronized void add(SSTableReader reader)
{
int level = reader.getSSTableLevel();
@@ -307,6 +338,11 @@ public class LeveledManifest
// This isn't a magic wand -- if you are consistently writing too fast for LCS to keep
// up, you're still screwed. But if instead you have intermittent bursts of activity,
// it can help a lot.
+ if (!calculatedLastCompactedKeys)
+ {
+ this.calculateLastCompactedKeys();
+ }
+
for (int i = generations.length - 1; i > 0; i--)
{
List<SSTableReader> sstables = getLevel(i);
[3/5] cassandra git commit: Disk failure policy should not be invoked
on out of space
Posted by ma...@apache.org.
Disk failure policy should not be invoked on out of space
patch by Sankalp Kohli; reviewed by marcuse for CASSANDRA-12385
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8bd62994
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8bd62994
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8bd62994
Branch: refs/heads/trunk
Commit: 8bd629942ca7014ff5b6e001db10290691f6cc08
Parents: 6808e75
Author: sankalp kohli <sa...@apple.com>
Authored: Fri Aug 19 09:44:48 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 23 16:32:11 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/Directories.java | 3 +-
.../db/compaction/AbstractCompactionTask.java | 7 +++++
.../cassandra/io/FSDiskFullWriteError.java | 33 ++++++++++++++++++++
4 files changed, 43 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8bd62994/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0d461f3..bd1fc5d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.9
+ * Disk failure policy should not be invoked on out of space (CASSANDRA-12385)
* Calculate last compacted key on startup (CASSANDRA-6216)
* Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190)
* Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8bd62994/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 30fe56d..68aa6be 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -47,6 +47,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.FSDiskFullWriteError;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.FileUtils;
@@ -385,7 +386,7 @@ public class Directories
if (candidates.isEmpty())
if (tooBig)
- throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
+ throw new FSDiskFullWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
else
throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8bd62994/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
index 155bf2f..430c916 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
@@ -23,6 +23,7 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorStatsCollector;
import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.io.FSDiskFullWriteError;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
@@ -59,6 +60,12 @@ public abstract class AbstractCompactionTask extends WrappedRunnable
{
return executeInternal(collector);
}
+ catch(FSDiskFullWriteError e)
+ {
+ RuntimeException cause = new RuntimeException("Converted from FSDiskFullWriteError: " + e.getMessage());
+ cause.setStackTrace(e.getStackTrace());
+ throw new RuntimeException("Throwing new Runtime to bypass exception handler when disk is full", cause);
+ }
finally
{
transaction.close();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8bd62994/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java b/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java
new file mode 100644
index 0000000..ca5d8da
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/FSDiskFullWriteError.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io;
+
+public class FSDiskFullWriteError extends FSWriteError
+{
+ public FSDiskFullWriteError(Throwable cause, String path)
+ {
+ super(cause, path);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "FSDiskFullWriteError in " + path;
+ }
+}