You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2015/06/24 01:17:37 UTC
[1/6] cassandra git commit: ColumnFamilyStore.selectAndReference may
block during compaction
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 b969496e4 -> 3caf0e029
refs/heads/cassandra-2.2 c889bab22 -> 0d4065e58
refs/heads/trunk 1902653e9 -> b7e72e1ee
ColumnFamilyStore.selectAndReference may block during compaction
patch by benedict; reviewed by aleksey for CASSANDRA-9637
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3caf0e02
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3caf0e02
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3caf0e02
Branch: refs/heads/cassandra-2.1
Commit: 3caf0e02970b8da90d154d81d769662057d7c772
Parents: b969496
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Jun 24 00:13:55 2015 +0100
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Wed Jun 24 00:13:55 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 15 +++++++++++++++
.../cassandra/db/compaction/CompactionTask.java | 4 +++-
3 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3caf0e02/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 28b145b..b3c76ed 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.7
+ * ColumnFamilyStore.selectAndReference may block during compaction (CASSANDRA-9637)
* Fix bug in cardinality check when compacting (CASSANDRA-9580)
* Fix memory leak in Ref due to ConcurrentLinkedQueue.remove() behaviour (CASSANDRA-9549)
Merged from 2.0
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3caf0e02/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index bd39409..a01bb49 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1851,12 +1851,27 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public RefViewFragment selectAndReference(Function<DataTracker.View, List<SSTableReader>> filter)
{
+ long failingSince = -1L;
while (true)
{
ViewFragment view = select(filter);
Refs<SSTableReader> refs = Refs.tryRef(view.sstables);
if (refs != null)
return new RefViewFragment(view.sstables, view.memtables, refs);
+ if (failingSince <= 0)
+ {
+ failingSince = System.nanoTime();
+ }
+ else if (TimeUnit.MILLISECONDS.toNanos(100) > System.nanoTime() - failingSince)
+ {
+ List<SSTableReader> released = new ArrayList<>();
+ for (SSTableReader reader : view.sstables)
+ if (reader.selfRef().globalCount() == 0)
+ released.add(reader);
+ logger.info("Spinning trying to capture released readers {}", released);
+ logger.info("Spinning trying to capture all readers {}", view.sstables);
+ failingSince = System.nanoTime();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3caf0e02/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 392034c..751f8f3 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -45,6 +45,7 @@ import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.concurrent.Refs;
public class CompactionTask extends AbstractCompactionTask
{
@@ -159,7 +160,8 @@ public class CompactionTask extends AbstractCompactionTask
// SSTableScanners need to be closed before markCompactedSSTablesReplaced call as scanners contain references
// to both ifile and dfile and SSTR will throw deletion errors on Windows if it tries to delete before scanner is closed.
// See CASSANDRA-8019 and CASSANDRA-8399
- try (AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
+ try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
+ AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
{
ci = new CompactionIterable(compactionType, scanners.scanners, controller);
Iterator<AbstractCompactedRow> iter = ci.iterator();
[5/6] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by be...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/db/compaction/CompactionTask.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0d4065e5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0d4065e5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0d4065e5
Branch: refs/heads/cassandra-2.2
Commit: 0d4065e58ad34a5c6ef525aa72cb1cfe859a5b0a
Parents: c889bab 3caf0e0
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Jun 24 00:14:36 2015 +0100
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Wed Jun 24 00:14:36 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 15 +++++++++++++++
.../cassandra/db/compaction/CompactionTask.java | 4 +++-
3 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d4065e5/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index c0480d7,b3c76ed..7a32693
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,23 -1,7 +1,24 @@@
-2.1.7
+2.2
+ * Update cqlsh for UDFs (CASSANDRA-7556)
+ * Change Windows kernel default timer resolution (CASSANDRA-9634)
+ * Deprected sstable2json and json2sstable (CASSANDRA-9618)
+ * Allow native functions in user-defined aggregates (CASSANDRA-9542)
+ * Don't repair system_distributed by default (CASSANDRA-9621)
+ * Fix mixing min, max, and count aggregates for blob type (CASSANRA-9622)
+ * Rename class for DATE type in Java driver (CASSANDRA-9563)
+ * Duplicate compilation of UDFs on coordinator (CASSANDRA-9475)
+ * Fix connection leak in CqlRecordWriter (CASSANDRA-9576)
+ * Mlockall before opening system sstables & remove boot_without_jna option (CASSANDRA-9573)
+ * Add functions to convert timeuuid to date or time, deprecate dateOf and unixTimestampOf (CASSANDRA-9229)
+ * Make sure we cancel non-compacting sstables from LifecycleTransaction (CASSANDRA-9566)
+ * Fix deprecated repair JMX API (CASSANDRA-9570)
+ * Add logback metrics (CASSANDRA-9378)
+ * Update and refactor ant test/test-compression to run the tests in parallel (CASSANDRA-9583)
+Merged from 2.1:
+ * ColumnFamilyStore.selectAndReference may block during compaction (CASSANDRA-9637)
* Fix bug in cardinality check when compacting (CASSANDRA-9580)
* Fix memory leak in Ref due to ConcurrentLinkedQueue.remove() behaviour (CASSANDRA-9549)
+ * Make rebuild only run one at a time (CASSANDRA-9119)
Merged from 2.0
* 'WITH WITH' in alter keyspace statements causes NPE (CASSANDRA-9565)
* Expose some internals of SelectStatement for inspection (CASSANDRA-9532)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d4065e5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index e9512f5,a01bb49..cd4b831
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -1823,9 -1849,9 +1823,10 @@@ public class ColumnFamilyStore implemen
return repairedSSTables;
}
- public RefViewFragment selectAndReference(Function<DataTracker.View, List<SSTableReader>> filter)
+ @SuppressWarnings("resource")
+ public RefViewFragment selectAndReference(Function<View, List<SSTableReader>> filter)
{
+ long failingSince = -1L;
while (true)
{
ViewFragment view = select(filter);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d4065e5/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 7089016,751f8f3..6aa59e4
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@@ -42,10 -39,13 +42,11 @@@ import org.apache.cassandra.config.Data
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorStatsCollector;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableRewriter;
-import org.apache.cassandra.io.sstable.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.UUIDGen;
+ import org.apache.cassandra.utils.concurrent.Refs;
public class CompactionTask extends AbstractCompactionTask
{
@@@ -157,36 -160,46 +158,37 @@@
// SSTableScanners need to be closed before markCompactedSSTablesReplaced call as scanners contain references
// to both ifile and dfile and SSTR will throw deletion errors on Windows if it tries to delete before scanner is closed.
// See CASSANDRA-8019 and CASSANDRA-8399
- try (AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
+ try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
+ AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
{
- ci = new CompactionIterable(compactionType, scanners.scanners, controller);
- Iterator<AbstractCompactedRow> iter = ci.iterator();
- // we can't preheat until the tracker has been set. This doesn't happen until we tell the cfs to
- // replace the old entries. Track entries to preheat here until then.
- long minRepairedAt = getMinRepairedAt(actuallyCompact);
- // we only need the age of the data that we're actually retaining
- long maxAge = getMaxDataAge(actuallyCompact);
- if (collector != null)
- collector.beginCompaction(ci);
- long lastCheckObsoletion = start;
- SSTableRewriter writer = new SSTableRewriter(cfs, sstables, maxAge, offline);
- try
+ ci = new CompactionIterable(compactionType, scanners.scanners, controller, sstableFormat, taskId);
+ try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator())
{
+ if (collector != null)
+ collector.beginCompaction(ci);
+ long lastCheckObsoletion = start;
+
if (!controller.cfs.getCompactionStrategy().isActive)
- throw new CompactionInterruptedException(ci.getCompactionInfo());
- if (!iter.hasNext())
- {
- // don't mark compacted in the finally block, since if there _is_ nondeleted data,
- // we need to sync it (via closeAndOpen) first, so there is no period during which
- // a crash could cause data loss.
- cfs.markObsolete(sstables, compactionType);
- return;
- }
+ throw new CompactionInterruptedException(ci.getCompactionInfo());
- writer.switchWriter(createCompactionWriter(cfs.directories.getLocationForDisk(getWriteDirectory(expectedSSTableSize)), keysPerSSTable, minRepairedAt));
- while (iter.hasNext())
+ try (CompactionAwareWriter writer = getCompactionAwareWriter(cfs, transaction, actuallyCompact))
{
- if (ci.isStopRequested())
- throw new CompactionInterruptedException(ci.getCompactionInfo());
-
- AbstractCompactedRow row = iter.next();
- if (writer.append(row) != null)
+ estimatedKeys = writer.estimatedKeys();
+ while (iter.hasNext())
{
- totalKeysWritten++;
- if (newSSTableSegmentThresholdReached(writer.currentWriter()))
+ if (ci.isStopRequested())
+ throw new CompactionInterruptedException(ci.getCompactionInfo());
+
+ try (AbstractCompactedRow row = iter.next())
{
- writer.switchWriter(createCompactionWriter(cfs.directories.getLocationForDisk(getWriteDirectory(expectedSSTableSize)), keysPerSSTable, minRepairedAt));
+ if (writer.append(row))
+ totalKeysWritten++;
+
+ if (System.nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L))
+ {
+ controller.maybeRefreshOverlaps();
+ lastCheckObsoletion = System.nanoTime();
+ }
}
}
[2/6] cassandra git commit: ColumnFamilyStore.selectAndReference may
block during compaction
Posted by be...@apache.org.
ColumnFamilyStore.selectAndReference may block during compaction
patch by benedict; reviewed by aleksey for CASSANDRA-9637
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3caf0e02
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3caf0e02
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3caf0e02
Branch: refs/heads/cassandra-2.2
Commit: 3caf0e02970b8da90d154d81d769662057d7c772
Parents: b969496
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Jun 24 00:13:55 2015 +0100
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Wed Jun 24 00:13:55 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 15 +++++++++++++++
.../cassandra/db/compaction/CompactionTask.java | 4 +++-
3 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3caf0e02/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 28b145b..b3c76ed 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.7
+ * ColumnFamilyStore.selectAndReference may block during compaction (CASSANDRA-9637)
* Fix bug in cardinality check when compacting (CASSANDRA-9580)
* Fix memory leak in Ref due to ConcurrentLinkedQueue.remove() behaviour (CASSANDRA-9549)
Merged from 2.0
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3caf0e02/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index bd39409..a01bb49 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1851,12 +1851,27 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public RefViewFragment selectAndReference(Function<DataTracker.View, List<SSTableReader>> filter)
{
+ long failingSince = -1L;
while (true)
{
ViewFragment view = select(filter);
Refs<SSTableReader> refs = Refs.tryRef(view.sstables);
if (refs != null)
return new RefViewFragment(view.sstables, view.memtables, refs);
+ if (failingSince <= 0)
+ {
+ failingSince = System.nanoTime();
+ }
+ else if (TimeUnit.MILLISECONDS.toNanos(100) > System.nanoTime() - failingSince)
+ {
+ List<SSTableReader> released = new ArrayList<>();
+ for (SSTableReader reader : view.sstables)
+ if (reader.selfRef().globalCount() == 0)
+ released.add(reader);
+ logger.info("Spinning trying to capture released readers {}", released);
+ logger.info("Spinning trying to capture all readers {}", view.sstables);
+ failingSince = System.nanoTime();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3caf0e02/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 392034c..751f8f3 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -45,6 +45,7 @@ import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.concurrent.Refs;
public class CompactionTask extends AbstractCompactionTask
{
@@ -159,7 +160,8 @@ public class CompactionTask extends AbstractCompactionTask
// SSTableScanners need to be closed before markCompactedSSTablesReplaced call as scanners contain references
// to both ifile and dfile and SSTR will throw deletion errors on Windows if it tries to delete before scanner is closed.
// See CASSANDRA-8019 and CASSANDRA-8399
- try (AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
+ try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
+ AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
{
ci = new CompactionIterable(compactionType, scanners.scanners, controller);
Iterator<AbstractCompactedRow> iter = ci.iterator();
[3/6] cassandra git commit: ColumnFamilyStore.selectAndReference may
block during compaction
Posted by be...@apache.org.
ColumnFamilyStore.selectAndReference may block during compaction
patch by benedict; reviewed by aleksey for CASSANDRA-9637
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3caf0e02
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3caf0e02
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3caf0e02
Branch: refs/heads/trunk
Commit: 3caf0e02970b8da90d154d81d769662057d7c772
Parents: b969496
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Jun 24 00:13:55 2015 +0100
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Wed Jun 24 00:13:55 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 15 +++++++++++++++
.../cassandra/db/compaction/CompactionTask.java | 4 +++-
3 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3caf0e02/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 28b145b..b3c76ed 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.7
+ * ColumnFamilyStore.selectAndReference may block during compaction (CASSANDRA-9637)
* Fix bug in cardinality check when compacting (CASSANDRA-9580)
* Fix memory leak in Ref due to ConcurrentLinkedQueue.remove() behaviour (CASSANDRA-9549)
Merged from 2.0
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3caf0e02/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index bd39409..a01bb49 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1851,12 +1851,27 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public RefViewFragment selectAndReference(Function<DataTracker.View, List<SSTableReader>> filter)
{
+ long failingSince = -1L;
while (true)
{
ViewFragment view = select(filter);
Refs<SSTableReader> refs = Refs.tryRef(view.sstables);
if (refs != null)
return new RefViewFragment(view.sstables, view.memtables, refs);
+ if (failingSince <= 0)
+ {
+ failingSince = System.nanoTime();
+ }
+ else if (TimeUnit.MILLISECONDS.toNanos(100) > System.nanoTime() - failingSince)
+ {
+ List<SSTableReader> released = new ArrayList<>();
+ for (SSTableReader reader : view.sstables)
+ if (reader.selfRef().globalCount() == 0)
+ released.add(reader);
+ logger.info("Spinning trying to capture released readers {}", released);
+ logger.info("Spinning trying to capture all readers {}", view.sstables);
+ failingSince = System.nanoTime();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3caf0e02/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 392034c..751f8f3 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -45,6 +45,7 @@ import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.concurrent.Refs;
public class CompactionTask extends AbstractCompactionTask
{
@@ -159,7 +160,8 @@ public class CompactionTask extends AbstractCompactionTask
// SSTableScanners need to be closed before markCompactedSSTablesReplaced call as scanners contain references
// to both ifile and dfile and SSTR will throw deletion errors on Windows if it tries to delete before scanner is closed.
// See CASSANDRA-8019 and CASSANDRA-8399
- try (AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
+ try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
+ AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
{
ci = new CompactionIterable(compactionType, scanners.scanners, controller);
Iterator<AbstractCompactedRow> iter = ci.iterator();
[4/6] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by be...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/db/compaction/CompactionTask.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0d4065e5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0d4065e5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0d4065e5
Branch: refs/heads/trunk
Commit: 0d4065e58ad34a5c6ef525aa72cb1cfe859a5b0a
Parents: c889bab 3caf0e0
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Jun 24 00:14:36 2015 +0100
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Wed Jun 24 00:14:36 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 15 +++++++++++++++
.../cassandra/db/compaction/CompactionTask.java | 4 +++-
3 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d4065e5/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index c0480d7,b3c76ed..7a32693
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,23 -1,7 +1,24 @@@
-2.1.7
+2.2
+ * Update cqlsh for UDFs (CASSANDRA-7556)
+ * Change Windows kernel default timer resolution (CASSANDRA-9634)
+ * Deprected sstable2json and json2sstable (CASSANDRA-9618)
+ * Allow native functions in user-defined aggregates (CASSANDRA-9542)
+ * Don't repair system_distributed by default (CASSANDRA-9621)
+ * Fix mixing min, max, and count aggregates for blob type (CASSANRA-9622)
+ * Rename class for DATE type in Java driver (CASSANDRA-9563)
+ * Duplicate compilation of UDFs on coordinator (CASSANDRA-9475)
+ * Fix connection leak in CqlRecordWriter (CASSANDRA-9576)
+ * Mlockall before opening system sstables & remove boot_without_jna option (CASSANDRA-9573)
+ * Add functions to convert timeuuid to date or time, deprecate dateOf and unixTimestampOf (CASSANDRA-9229)
+ * Make sure we cancel non-compacting sstables from LifecycleTransaction (CASSANDRA-9566)
+ * Fix deprecated repair JMX API (CASSANDRA-9570)
+ * Add logback metrics (CASSANDRA-9378)
+ * Update and refactor ant test/test-compression to run the tests in parallel (CASSANDRA-9583)
+Merged from 2.1:
+ * ColumnFamilyStore.selectAndReference may block during compaction (CASSANDRA-9637)
* Fix bug in cardinality check when compacting (CASSANDRA-9580)
* Fix memory leak in Ref due to ConcurrentLinkedQueue.remove() behaviour (CASSANDRA-9549)
+ * Make rebuild only run one at a time (CASSANDRA-9119)
Merged from 2.0
* 'WITH WITH' in alter keyspace statements causes NPE (CASSANDRA-9565)
* Expose some internals of SelectStatement for inspection (CASSANDRA-9532)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d4065e5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index e9512f5,a01bb49..cd4b831
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -1823,9 -1849,9 +1823,10 @@@ public class ColumnFamilyStore implemen
return repairedSSTables;
}
- public RefViewFragment selectAndReference(Function<DataTracker.View, List<SSTableReader>> filter)
+ @SuppressWarnings("resource")
+ public RefViewFragment selectAndReference(Function<View, List<SSTableReader>> filter)
{
+ long failingSince = -1L;
while (true)
{
ViewFragment view = select(filter);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0d4065e5/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 7089016,751f8f3..6aa59e4
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@@ -42,10 -39,13 +42,11 @@@ import org.apache.cassandra.config.Data
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorStatsCollector;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableRewriter;
-import org.apache.cassandra.io.sstable.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.UUIDGen;
+ import org.apache.cassandra.utils.concurrent.Refs;
public class CompactionTask extends AbstractCompactionTask
{
@@@ -157,36 -160,46 +158,37 @@@
// SSTableScanners need to be closed before markCompactedSSTablesReplaced call as scanners contain references
// to both ifile and dfile and SSTR will throw deletion errors on Windows if it tries to delete before scanner is closed.
// See CASSANDRA-8019 and CASSANDRA-8399
- try (AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
+ try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
+ AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
{
- ci = new CompactionIterable(compactionType, scanners.scanners, controller);
- Iterator<AbstractCompactedRow> iter = ci.iterator();
- // we can't preheat until the tracker has been set. This doesn't happen until we tell the cfs to
- // replace the old entries. Track entries to preheat here until then.
- long minRepairedAt = getMinRepairedAt(actuallyCompact);
- // we only need the age of the data that we're actually retaining
- long maxAge = getMaxDataAge(actuallyCompact);
- if (collector != null)
- collector.beginCompaction(ci);
- long lastCheckObsoletion = start;
- SSTableRewriter writer = new SSTableRewriter(cfs, sstables, maxAge, offline);
- try
+ ci = new CompactionIterable(compactionType, scanners.scanners, controller, sstableFormat, taskId);
+ try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator())
{
+ if (collector != null)
+ collector.beginCompaction(ci);
+ long lastCheckObsoletion = start;
+
if (!controller.cfs.getCompactionStrategy().isActive)
- throw new CompactionInterruptedException(ci.getCompactionInfo());
- if (!iter.hasNext())
- {
- // don't mark compacted in the finally block, since if there _is_ nondeleted data,
- // we need to sync it (via closeAndOpen) first, so there is no period during which
- // a crash could cause data loss.
- cfs.markObsolete(sstables, compactionType);
- return;
- }
+ throw new CompactionInterruptedException(ci.getCompactionInfo());
- writer.switchWriter(createCompactionWriter(cfs.directories.getLocationForDisk(getWriteDirectory(expectedSSTableSize)), keysPerSSTable, minRepairedAt));
- while (iter.hasNext())
+ try (CompactionAwareWriter writer = getCompactionAwareWriter(cfs, transaction, actuallyCompact))
{
- if (ci.isStopRequested())
- throw new CompactionInterruptedException(ci.getCompactionInfo());
-
- AbstractCompactedRow row = iter.next();
- if (writer.append(row) != null)
+ estimatedKeys = writer.estimatedKeys();
+ while (iter.hasNext())
{
- totalKeysWritten++;
- if (newSSTableSegmentThresholdReached(writer.currentWriter()))
+ if (ci.isStopRequested())
+ throw new CompactionInterruptedException(ci.getCompactionInfo());
+
+ try (AbstractCompactedRow row = iter.next())
{
- writer.switchWriter(createCompactionWriter(cfs.directories.getLocationForDisk(getWriteDirectory(expectedSSTableSize)), keysPerSSTable, minRepairedAt));
+ if (writer.append(row))
+ totalKeysWritten++;
+
+ if (System.nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L))
+ {
+ controller.maybeRefreshOverlaps();
+ lastCheckObsoletion = System.nanoTime();
+ }
}
}
[6/6] cassandra git commit: Merge branch 'cassandra-2.2' into trunk
Posted by be...@apache.org.
Merge branch 'cassandra-2.2' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b7e72e1e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b7e72e1e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b7e72e1e
Branch: refs/heads/trunk
Commit: b7e72e1eefcd883c2cbc757ae42e455699d50bfd
Parents: 1902653 0d4065e
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Jun 24 00:14:49 2015 +0100
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Wed Jun 24 00:14:49 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamilyStore.java | 15 +++++++++++++++
.../cassandra/db/compaction/CompactionTask.java | 4 +++-
3 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7e72e1e/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7e72e1e/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7e72e1e/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 17e55bf,6aa59e4..7dbeb44
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@@ -157,9 -158,9 +158,10 @@@ public class CompactionTask extends Abs
// SSTableScanners need to be closed before markCompactedSSTablesReplaced call as scanners contain references
// to both ifile and dfile and SSTR will throw deletion errors on Windows if it tries to delete before scanner is closed.
// See CASSANDRA-8019 and CASSANDRA-8399
- try (AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
+ try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
+ AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
{
+
ci = new CompactionIterable(compactionType, scanners.scanners, controller, sstableFormat, taskId);
try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator())
{