You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2015/03/31 12:54:28 UTC
[1/2] cassandra git commit: Don't wait for min_threshold sstables in
the same window in DTCS
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 a2b66bf10 -> 994eb2fa7
Don't wait for min_threshold sstables in the same window in DTCS
Patch by Björn Hegerfors; reviewed by marcuse for CASSANDRA-8360
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f7116c91
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f7116c91
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f7116c91
Branch: refs/heads/cassandra-2.1
Commit: f7116c91b237d920249e5bcae653b47631f206c9
Parents: c94da6c
Author: Björn Hegerfors <bj...@spotify.com>
Authored: Tue Mar 31 12:41:27 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 31 12:51:08 2015 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../compaction/DateTieredCompactionStrategy.java | 18 ++++++++++++++----
.../DateTieredCompactionStrategyTest.java | 7 +++++--
3 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7116c91/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ed40a78..0bcc5cb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.14:
+ * Do more aggressive compaction in old time windows in DTCS (CASSANDRA-8360)
* java.lang.AssertionError when reading saved cache (CASSANDRA-8740)
* "disk full" when running cleanup (CASSANDRA-9036)
* Make PasswordAuthenticator number of hashing rounds configurable (CASSANDRA-8085)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7116c91/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index 9c708db..6b3e800 100644
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -103,7 +103,11 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
List<List<SSTableReader>> buckets = getBuckets(createSSTableAndMinTimestampPairs(candidates), options.baseTime, base, now);
logger.debug("Compaction buckets are {}", buckets);
updateEstimatedCompactionsByTasks(buckets);
- List<SSTableReader> mostInteresting = newestBucket(buckets, cfs.getMinimumCompactionThreshold(), cfs.getMaximumCompactionThreshold());
+ List<SSTableReader> mostInteresting = newestBucket(buckets,
+ cfs.getMinimumCompactionThreshold(),
+ cfs.getMaximumCompactionThreshold(),
+ options.baseTime,
+ now);
if (!mostInteresting.isEmpty())
return mostInteresting;
return null;
@@ -298,12 +302,18 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
* @return a bucket (list) of sstables to compact.
*/
@VisibleForTesting
- static List<SSTableReader> newestBucket(List<List<SSTableReader>> buckets, int minThreshold, int maxThreshold)
+ static List<SSTableReader> newestBucket(List<List<SSTableReader>> buckets, int minThreshold, int maxThreshold, long now, long baseTime)
{
- // Skip buckets containing less than minThreshold sstables, and limit other buckets to maxThreshold sstables.
+ // If the "incoming window" has at least minThreshold SSTables, choose that one.
+ // For any other bucket, at least 2 SSTables is enough.
+ // In any case, limit to maxThreshold SSTables.
+ Target incomingWindow = getInitialTarget(now, baseTime);
for (List<SSTableReader> bucket : buckets)
- if (bucket.size() >= minThreshold)
+ {
+ if (bucket.size() >= minThreshold ||
+ (bucket.size() >= 2 && !incomingWindow.onTarget(bucket.get(0).getMinTimestamp())))
return trimToThreshold(bucket, maxThreshold);
+ }
return Collections.emptyList();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7116c91/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index 7d48575..f98e372 100644
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@ -213,8 +213,11 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader
List<SSTableReader> sstrs = new ArrayList<>(cfs.getSSTables());
- List<SSTableReader> newBucket = newestBucket(Collections.singletonList(sstrs.subList(0, 2)), 4, 32);
- assertTrue("nothing should be returned when all buckets are below the min threshold", newBucket.isEmpty());
+ List<SSTableReader> newBucket = newestBucket(Collections.singletonList(sstrs.subList(0, 2)), 4, 32, 9, 10);
+ assertTrue("incoming bucket should not be accepted when it has below the min threshold SSTables", newBucket.isEmpty());
+
+ newBucket = newestBucket(Collections.singletonList(sstrs.subList(0, 2)), 4, 32, 10, 10);
+ assertFalse("non-incoming bucket should be accepted when it has at least 2 SSTables", newBucket.isEmpty());
assertEquals("an sstable with a single value should have equal min/max timestamps", sstrs.get(0).getMinTimestamp(), sstrs.get(0).getMaxTimestamp());
assertEquals("an sstable with a single value should have equal min/max timestamps", sstrs.get(1).getMinTimestamp(), sstrs.get(1).getMaxTimestamp());
[2/2] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Posted by ma...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/994eb2fa
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/994eb2fa
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/994eb2fa
Branch: refs/heads/cassandra-2.1
Commit: 994eb2fa71f802f72fc1b06f36398e5a98467578
Parents: a2b66bf f7116c9
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 31 12:53:03 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 31 12:53:03 2015 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../compaction/DateTieredCompactionStrategy.java | 18 ++++++++++++++----
.../DateTieredCompactionStrategyTest.java | 7 +++++--
3 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/994eb2fa/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index e3b5363,0bcc5cb..e22c6b3
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,71 -1,8 +1,72 @@@
-2.0.14:
+2.1.4
+ * Buffer bloom filter serialization (CASSANDRA-9066)
+ * Fix anti-compaction target bloom filter size (CASSANDRA-9060)
+ * Make FROZEN and TUPLE unreserved keywords in CQL (CASSANDRA-9047)
+ * Prevent AssertionError from SizeEstimatesRecorder (CASSANDRA-9034)
+ * Avoid overwriting index summaries for sstables with an older format that
+ does not support downsampling; rebuild summaries on startup when this
+ is detected (CASSANDRA-8993)
+ * Fix potential data loss in CompressedSequentialWriter (CASSANDRA-8949)
+ * Make PasswordAuthenticator number of hashing rounds configurable (CASSANDRA-8085)
+ * Fix AssertionError when binding nested collections in DELETE (CASSANDRA-8900)
+ * Check for overlap with non-early sstables in LCS (CASSANDRA-8739)
+ * Only calculate max purgable timestamp if we have to (CASSANDRA-8914)
+ * (cqlsh) Greatly improve performance of COPY FROM (CASSANDRA-8225)
+ * IndexSummary effectiveIndexInterval is now a guideline, not a rule (CASSANDRA-8993)
+ * Use correct bounds for page cache eviction of compressed files (CASSANDRA-8746)
+ * SSTableScanner enforces its bounds (CASSANDRA-8946)
+ * Cleanup cell equality (CASSANDRA-8947)
+ * Introduce intra-cluster message coalescing (CASSANDRA-8692)
+ * DatabaseDescriptor throws NPE when rpc_interface is used (CASSANDRA-8839)
+ * Don't check if an sstable is live for offline compactions (CASSANDRA-8841)
+ * Don't set clientMode in SSTableLoader (CASSANDRA-8238)
+ * Fix SSTableRewriter with disabled early open (CASSANDRA-8535)
+ * Allow invalidating permissions and cache time (CASSANDRA-8722)
+ * Log warning when queries that will require ALLOW FILTERING in Cassandra 3.0
+ are executed (CASSANDRA-8418)
+ * Fix cassandra-stress so it respects the CL passed in user mode (CASSANDRA-8948)
+ * Fix rare NPE in ColumnDefinition#hasIndexOption() (CASSANDRA-8786)
+ * cassandra-stress reports per-operation statistics, plus misc (CASSANDRA-8769)
+ * Add SimpleDate (cql date) and Time (cql time) types (CASSANDRA-7523)
+ * Use long for key count in cfstats (CASSANDRA-8913)
+ * Make SSTableRewriter.abort() more robust to failure (CASSANDRA-8832)
+ * Remove cold_reads_to_omit from STCS (CASSANDRA-8860)
+ * Make EstimatedHistogram#percentile() use ceil instead of floor (CASSANDRA-8883)
+ * Fix top partitions reporting wrong cardinality (CASSANDRA-8834)
+ * Fix rare NPE in KeyCacheSerializer (CASSANDRA-8067)
+ * Pick sstables for validation as late as possible inc repairs (CASSANDRA-8366)
+ * Fix commitlog getPendingTasks to not increment (CASSANDRA-8856)
+ * Fix parallelism adjustment in range and secondary index queries
+ when the first fetch does not satisfy the limit (CASSANDRA-8856)
+ * Check if the filtered sstables is non-empty in STCS (CASSANDRA-8843)
+ * Upgrade java-driver used for cassandra-stress (CASSANDRA-8842)
+ * Fix CommitLog.forceRecycleAllSegments() memory access error (CASSANDRA-8812)
+ * Improve assertions in Memory (CASSANDRA-8792)
+ * Fix SSTableRewriter cleanup (CASSANDRA-8802)
+ * Introduce SafeMemory for CompressionMetadata.Writer (CASSANDRA-8758)
+ * 'nodetool info' prints exception against older node (CASSANDRA-8796)
+ * Ensure SSTableReader.last corresponds exactly with the file end (CASSANDRA-8750)
+ * Make SSTableWriter.openEarly more robust and obvious (CASSANDRA-8747)
+ * Enforce SSTableReader.first/last (CASSANDRA-8744)
+ * Cleanup SegmentedFile API (CASSANDRA-8749)
+ * Avoid overlap with early compaction replacement (CASSANDRA-8683)
+ * Safer Resource Management++ (CASSANDRA-8707)
+ * Write partition size estimates into a system table (CASSANDRA-7688)
+ * cqlsh: Fix keys() and full() collection indexes in DESCRIBE output
+ (CASSANDRA-8154)
+ * Show progress of streaming in nodetool netstats (CASSANDRA-8886)
+ * IndexSummaryBuilder utilises offheap memory, and shares data between
+ each IndexSummary opened from it (CASSANDRA-8757)
+ * markCompacting only succeeds if the exact SSTableReader instances being
+ marked are in the live set (CASSANDRA-8689)
+ * cassandra-stress support for varint (CASSANDRA-8882)
+ * Fix Adler32 digest for compressed sstables (CASSANDRA-8778)
+ * Add nodetool statushandoff/statusbackup (CASSANDRA-8912)
+ * Use stdout for progress and stats in sstableloader (CASSANDRA-8982)
+Merged from 2.0:
+ * Do more aggressive compaction in old time windows in DTCS (CASSANDRA-8360)
* java.lang.AssertionError when reading saved cache (CASSANDRA-8740)
* "disk full" when running cleanup (CASSANDRA-9036)
- * Make PasswordAuthenticator number of hashing rounds configurable (CASSANDRA-8085)
* Lower logging level from ERROR to DEBUG when a scheduled schema pull
cannot be completed due to a node being down (CASSANDRA-9032)
* Fix MOVED_NODE client event (CASSANDRA-8516)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/994eb2fa/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/994eb2fa/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------