You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2015/08/18 11:02:58 UTC
[1/5] cassandra git commit: Add option to change how often we check
for expired sstables with DTCS
Repository: cassandra
Updated Branches:
refs/heads/trunk 30a704e01 -> 91e3d3421
Add option to change how often we check for expired sstables with DTCS
Patch by marcuse; reviewed by yukim for CASSANDRA-9882
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/17042f19
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/17042f19
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/17042f19
Branch: refs/heads/trunk
Commit: 17042f19b084d93bd3bcc35c7282812f456b7640
Parents: f53aacb
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Aug 17 09:50:37 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 18 10:47:04 2015 +0200
----------------------------------------------------------------------
.../DateTieredCompactionStrategy.java | 7 +++----
.../DateTieredCompactionStrategyOptions.java | 21 ++++++++++++++++++++
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index fea4995..2e43352 100644
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -38,8 +38,7 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
protected DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
- @VisibleForTesting
- long lastExpiredCheck;
+ private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
{
@@ -87,8 +86,8 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
Set<SSTableReader> uncompacting = cfs.getUncompactingSSTables();
Set<SSTableReader> expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes due to it being an expensive operation
- if (System.currentTimeMillis() - lastExpiredCheck > TimeUnit.MINUTES.toMillis(10))
+ // we only check for expired sstables every 10 minutes (by default) due to it being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck > options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter what.
expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
index f54c020..0cbf90e 100644
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
@@ -27,12 +27,15 @@ public final class DateTieredCompactionStrategyOptions
protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION = TimeUnit.MICROSECONDS;
protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365;
protected static final long DEFAULT_BASE_TIME_SECONDS = 60;
+ protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS = 60 * 10;
protected static final String TIMESTAMP_RESOLUTION_KEY = "timestamp_resolution";
protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days";
protected static final String BASE_TIME_KEY = "base_time_seconds";
+ protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY = "expired_sstable_check_frequency_seconds";
protected final long maxSSTableAge;
protected final long baseTime;
+ protected final long expiredSSTableCheckFrequency;
public DateTieredCompactionStrategyOptions(Map<String, String> options)
{
@@ -43,12 +46,15 @@ public final class DateTieredCompactionStrategyOptions
maxSSTableAge = Math.round(fractionalDays * timestampResolution.convert(1, TimeUnit.DAYS));
optionValue = options.get(BASE_TIME_KEY);
baseTime = timestampResolution.convert(optionValue == null ? DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
+ optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
+ expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
}
public DateTieredCompactionStrategyOptions()
{
maxSSTableAge = Math.round(DEFAULT_MAX_SSTABLE_AGE_DAYS * DEFAULT_TIMESTAMP_RESOLUTION.convert(1, TimeUnit.DAYS));
baseTime = DEFAULT_TIMESTAMP_RESOLUTION.convert(DEFAULT_BASE_TIME_SECONDS, TimeUnit.SECONDS);
+ expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS, TimeUnit.SECONDS);
}
public static Map<String, String> validateOptions(Map<String, String> options, Map<String, String> uncheckedOptions) throws ConfigurationException
@@ -92,9 +98,24 @@ public final class DateTieredCompactionStrategyOptions
throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, BASE_TIME_KEY), e);
}
+ optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
+ try
+ {
+ long expiredCheckFrequency = optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue);
+ if (expiredCheckFrequency < 0)
+ {
+ throw new ConfigurationException(String.format("%s must not be negative, but was %d", EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, expiredCheckFrequency));
+ }
+ }
+ catch (NumberFormatException e)
+ {
+ throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY), e);
+ }
+
uncheckedOptions.remove(MAX_SSTABLE_AGE_KEY);
uncheckedOptions.remove(BASE_TIME_KEY);
uncheckedOptions.remove(TIMESTAMP_RESOLUTION_KEY);
+ uncheckedOptions.remove(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
return uncheckedOptions;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index 0084a16..6d9bfc2 100644
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@ -305,10 +305,10 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d / (24 * 60 * 60))));
+ options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, "0");
DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000)));
- dtcs.lastExpiredCheck = 0;
Thread.sleep(2000);
AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000));
assertNotNull(t);
[3/5] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/09147a8b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/09147a8b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/09147a8b
Branch: refs/heads/trunk
Commit: 09147a8b6c2ced97a05d185c02bedbf1a59978e1
Parents: ee186b6 755a6b2
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Aug 18 10:51:56 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 18 10:51:56 2015 +0200
----------------------------------------------------------------------
.../DateTieredCompactionStrategy.java | 7 +++----
.../DateTieredCompactionStrategyOptions.java | 21 ++++++++++++++++++++
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/09147a8b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/09147a8b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
[4/5] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6c69f07c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6c69f07c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6c69f07c
Branch: refs/heads/trunk
Commit: 6c69f07c2ed3a24398a55811f8c8cad326c3e2ce
Parents: 2350320 09147a8
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Aug 18 10:56:06 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 18 10:56:06 2015 +0200
----------------------------------------------------------------------
.../DateTieredCompactionStrategy.java | 7 +++----
.../DateTieredCompactionStrategyOptions.java | 21 ++++++++++++++++++++
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6c69f07c/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index a908884,0956962..ad14a9c
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -85,17 -81,17 +84,17 @@@ public class DateTieredCompactionStrate
*/
private List<SSTableReader> getNextBackgroundSSTables(final int gcBefore)
{
- if (cfs.getSSTables().isEmpty())
+ if (Iterables.isEmpty(cfs.getSSTables(SSTableSet.LIVE)))
return Collections.emptyList();
- Set<SSTableReader> uncompacting = Sets.intersection(sstables, cfs.getUncompactingSSTables());
+ Set<SSTableReader> uncompacting = ImmutableSet.copyOf(filter(cfs.getUncompactingSSTables(), sstables::contains));
Set<SSTableReader> expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes due to it being an expensive operation
- if (System.currentTimeMillis() - lastExpiredCheck > TimeUnit.MINUTES.toMillis(10))
+ // we only check for expired sstables every 10 minutes (by default) due to it being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck > options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter what.
- expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
+ expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(SSTableSet.CANONICAL, uncompacting), gcBefore);
lastExpiredCheck = System.currentTimeMillis();
}
Set<SSTableReader> candidates = Sets.newHashSet(filterSuspectSSTables(uncompacting));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6c69f07c/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index 7b3b9e7,0159c83..003a1cf
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -319,8 -316,9 +319,9 @@@ public class DateTieredCompactionStrate
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d / (24 * 60 * 60))));
+ options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, "0");
DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
- for (SSTableReader sstable : cfs.getSSTables())
+ for (SSTableReader sstable : cfs.getLiveSSTables())
dtcs.addSSTable(sstable);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000)));
[2/5] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Posted by ma...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/755a6b24
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/755a6b24
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/755a6b24
Branch: refs/heads/trunk
Commit: 755a6b244c7c0db34044f03b9a62c2070e033a78
Parents: 26ff150 17042f1
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Aug 18 10:51:38 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 18 10:51:38 2015 +0200
----------------------------------------------------------------------
.../DateTieredCompactionStrategy.java | 7 +++----
.../DateTieredCompactionStrategyOptions.java | 21 ++++++++++++++++++++
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index 41c304b,2e43352..a8e2aff
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -36,11 -36,9 +36,10 @@@ public class DateTieredCompactionStrate
{
private static final Logger logger = LoggerFactory.getLogger(DateTieredCompactionStrategy.class);
- protected DateTieredCompactionStrategyOptions options;
+ private final DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
+ private final Set<SSTableReader> sstables = new HashSet<>();
- @VisibleForTesting
- long lastExpiredCheck;
+ private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
{
@@@ -79,14 -80,14 +78,14 @@@
*/
private List<SSTableReader> getNextBackgroundSSTables(final int gcBefore)
{
- if (!isEnabled() || cfs.getSSTables().isEmpty())
+ if (cfs.getSSTables().isEmpty())
return Collections.emptyList();
- Set<SSTableReader> uncompacting = cfs.getUncompactingSSTables();
+ Set<SSTableReader> uncompacting = Sets.intersection(sstables, cfs.getUncompactingSSTables());
Set<SSTableReader> expired = Collections.emptySet();
- // we only check for expired sstables every 10 minutes due to it being an expensive operation
- if (System.currentTimeMillis() - lastExpiredCheck > TimeUnit.MINUTES.toMillis(10))
+ // we only check for expired sstables every 10 minutes (by default) due to it being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck > options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter what.
expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/755a6b24/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index cea835f,6d9bfc2..f05bf44
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -305,12 -305,10 +305,12 @@@ public class DateTieredCompactionStrate
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d / (24 * 60 * 60))));
+ options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, "0");
DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
+ for (SSTableReader sstable : cfs.getSSTables())
+ dtcs.addSSTable(sstable);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000)));
- dtcs.lastExpiredCheck = 0;
Thread.sleep(2000);
AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000));
assertNotNull(t);
[5/5] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/91e3d342
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/91e3d342
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/91e3d342
Branch: refs/heads/trunk
Commit: 91e3d34210e593f3ae03070865d1f346f3b3bac7
Parents: 30a704e 6c69f07
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Aug 18 10:59:53 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Aug 18 10:59:53 2015 +0200
----------------------------------------------------------------------
.../DateTieredCompactionStrategy.java | 7 +++----
.../DateTieredCompactionStrategyOptions.java | 21 ++++++++++++++++++++
.../DateTieredCompactionStrategyTest.java | 2 +-
3 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------