You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2017/12/11 08:07:09 UTC
[01/10] cassandra git commit: Grab refs during scrub,
index summary redistribution and cleanup
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.2 797de4ae3 -> 3cd2c3c4e
refs/heads/cassandra-3.0 a9225f90e -> d7329a639
refs/heads/cassandra-3.11 16bcbb925 -> 817f3c282
refs/heads/trunk 78150142e -> 1cb050922
Grab refs during scrub, index summary redistribution and cleanup
Patch by marcuse; reviewed by Joel Knighton for CASSANDRA-13873
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3cd2c3c4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3cd2c3c4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3cd2c3c4
Branch: refs/heads/cassandra-2.2
Commit: 3cd2c3c4ea4286562b2cb8443d6173ee251e6212
Parents: 797de4a
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Oct 23 09:43:44 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 08:53:44 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +-
.../cassandra/db/compaction/CompactionManager.java | 3 ++-
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 752cbdc..c1e81fd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,5 @@
2.2.12
- *
+ * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873)
2.2.11
* Safely handle empty buffers when outputting to JSON (CASSANDRA-13868)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index cd50646..2e69b6f 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -856,7 +856,8 @@ public class CompactionManager implements CompactionManagerMBean
metrics.beginCompaction(ci);
List<SSTableReader> finished;
try (SSTableRewriter writer = new SSTableRewriter(cfs, txn, sstable.maxDataAge, false);
- CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs)))
+ CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs));
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index aaed234..b6b20fb 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -40,6 +40,7 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.concurrent.Refs;
public class Scrubber implements Closeable
{
@@ -142,7 +143,8 @@ public class Scrubber implements Closeable
public void scrub()
{
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline()))
+ try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline());
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
index aad479b..12586e5 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
@@ -123,11 +124,14 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
logger.trace("Index summaries for compacting SSTables are using {} MB of space",
(memoryPoolBytes - remainingBytes) / 1024.0 / 1024.0);
- List<SSTableReader> newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
-
- for (LifecycleTransaction txn : transactions.values())
- txn.finish();
+ List<SSTableReader> newSSTables;
+ try (Refs<SSTableReader> refs = Refs.ref(sstablesByHotness))
+ {
+ newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
+ for (LifecycleTransaction txn : transactions.values())
+ txn.finish();
+ }
total = 0;
for (SSTableReader sstable : Iterables.concat(compacting, oldFormatSSTables, newSSTables))
total += sstable.getIndexSummaryOffHeapSize();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[09/10] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/817f3c28
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/817f3c28
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/817f3c28
Branch: refs/heads/trunk
Commit: 817f3c282debd8f00a7803bee4950cdc73ec7fcc
Parents: 16bcbb9 d7329a6
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Dec 11 09:03:55 2017 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 09:03:55 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../cassandra/db/compaction/CompactionManager.java | 1 +
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/817f3c28/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b7a6e14,8cff716..5faede2
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -26,22 -16,11 +26,24 @@@ Merged from 3.0
* Provide a JMX call to sync schema with local storage (CASSANDRA-13954)
* Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
* Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
+ Merged from 2.2:
+ * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873)
-3.0.15
+3.11.1
+ * Fix the computation of cdc_total_space_in_mb for exabyte filesystems (CASSANDRA-13808)
+ * AbstractTokenTreeBuilder#serializedSize returns wrong value when there is a single leaf and overflow collisions (CASSANDRA-13869)
+ * Add a compaction option to TWCS to ignore sstables overlapping checks (CASSANDRA-13418)
+ * BTree.Builder memory leak (CASSANDRA-13754)
+ * Revert CASSANDRA-10368 of supporting non-pk column filtering due to correctness (CASSANDRA-13798)
+ * Add a skip read validation flag to cassandra-stress (CASSANDRA-13772)
+ * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938)
+ * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744)
+ * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
+ * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
+ * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
+ * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
+Merged from 3.0:
* Improve TRUNCATE performance (CASSANDRA-13909)
* Implement short read protection on partition boundaries (CASSANDRA-13595)
* Fix ISE thrown by UPI.Serializer.hasNext() for some SELECT queries (CASSANDRA-13911)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/817f3c28/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 4030384,eaeb089..638e74c
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -1085,18 -865,16 +1085,19 @@@ public class CompactionManager implemen
logger.info("Cleaning up {}", sstable);
- File compactionFileLocation = cfs.getDirectories().getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(txn.originals(), OperationType.CLEANUP));
- if (compactionFileLocation == null)
- throw new IOException("disk full");
+ File compactionFileLocation = sstable.descriptor.directory;
+ RateLimiter limiter = getRateLimiter();
+ double compressionRatio = sstable.getCompressionRatio();
+ if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
+ compressionRatio = 1.0;
List<SSTableReader> finished;
+
int nowInSec = FBUtilities.nowInSeconds();
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge, false);
- ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
+ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge);
+ ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, null);
CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable));
CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics))
{
writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, txn));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/817f3c28/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 4635824,c8e0c53..c189c0b
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -141,8 -145,9 +142,9 @@@ public class Scrubber implements Closea
{
List<SSTableReader> finished = new ArrayList<>();
boolean completed = false;
- outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge, transaction.isOffline());
+ outputHandler.output(String.format("Scrubbing %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length())));
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge))
++ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge);
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/817f3c28/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
index 8fb4835,ebe98e5..73626d3
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
@@@ -40,8 -40,8 +40,9 @@@ import org.apache.cassandra.db.compacti
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
+ import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[08/10] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/817f3c28
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/817f3c28
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/817f3c28
Branch: refs/heads/cassandra-3.11
Commit: 817f3c282debd8f00a7803bee4950cdc73ec7fcc
Parents: 16bcbb9 d7329a6
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Dec 11 09:03:55 2017 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 09:03:55 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../cassandra/db/compaction/CompactionManager.java | 1 +
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/817f3c28/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b7a6e14,8cff716..5faede2
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -26,22 -16,11 +26,24 @@@ Merged from 3.0
* Provide a JMX call to sync schema with local storage (CASSANDRA-13954)
* Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
* Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
+ Merged from 2.2:
+ * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873)
-3.0.15
+3.11.1
+ * Fix the computation of cdc_total_space_in_mb for exabyte filesystems (CASSANDRA-13808)
+ * AbstractTokenTreeBuilder#serializedSize returns wrong value when there is a single leaf and overflow collisions (CASSANDRA-13869)
+ * Add a compaction option to TWCS to ignore sstables overlapping checks (CASSANDRA-13418)
+ * BTree.Builder memory leak (CASSANDRA-13754)
+ * Revert CASSANDRA-10368 of supporting non-pk column filtering due to correctness (CASSANDRA-13798)
+ * Add a skip read validation flag to cassandra-stress (CASSANDRA-13772)
+ * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938)
+ * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744)
+ * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
+ * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
+ * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
+ * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
+Merged from 3.0:
* Improve TRUNCATE performance (CASSANDRA-13909)
* Implement short read protection on partition boundaries (CASSANDRA-13595)
* Fix ISE thrown by UPI.Serializer.hasNext() for some SELECT queries (CASSANDRA-13911)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/817f3c28/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 4030384,eaeb089..638e74c
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -1085,18 -865,16 +1085,19 @@@ public class CompactionManager implemen
logger.info("Cleaning up {}", sstable);
- File compactionFileLocation = cfs.getDirectories().getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(txn.originals(), OperationType.CLEANUP));
- if (compactionFileLocation == null)
- throw new IOException("disk full");
+ File compactionFileLocation = sstable.descriptor.directory;
+ RateLimiter limiter = getRateLimiter();
+ double compressionRatio = sstable.getCompressionRatio();
+ if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
+ compressionRatio = 1.0;
List<SSTableReader> finished;
+
int nowInSec = FBUtilities.nowInSeconds();
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge, false);
- ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
+ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge);
+ ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, null);
CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable));
CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics))
{
writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, txn));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/817f3c28/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 4635824,c8e0c53..c189c0b
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -141,8 -145,9 +142,9 @@@ public class Scrubber implements Closea
{
List<SSTableReader> finished = new ArrayList<>();
boolean completed = false;
- outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge, transaction.isOffline());
+ outputHandler.output(String.format("Scrubbing %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length())));
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge))
++ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge);
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/817f3c28/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
index 8fb4835,ebe98e5..73626d3
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
@@@ -40,8 -40,8 +40,9 @@@ import org.apache.cassandra.db.compacti
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
+ import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[02/10] cassandra git commit: Grab refs during scrub,
index summary redistribution and cleanup
Posted by ma...@apache.org.
Grab refs during scrub, index summary redistribution and cleanup
Patch by marcuse; reviewed by Joel Knighton for CASSANDRA-13873
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3cd2c3c4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3cd2c3c4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3cd2c3c4
Branch: refs/heads/cassandra-3.0
Commit: 3cd2c3c4ea4286562b2cb8443d6173ee251e6212
Parents: 797de4a
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Oct 23 09:43:44 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 08:53:44 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +-
.../cassandra/db/compaction/CompactionManager.java | 3 ++-
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 752cbdc..c1e81fd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,5 @@
2.2.12
- *
+ * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873)
2.2.11
* Safely handle empty buffers when outputting to JSON (CASSANDRA-13868)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index cd50646..2e69b6f 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -856,7 +856,8 @@ public class CompactionManager implements CompactionManagerMBean
metrics.beginCompaction(ci);
List<SSTableReader> finished;
try (SSTableRewriter writer = new SSTableRewriter(cfs, txn, sstable.maxDataAge, false);
- CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs)))
+ CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs));
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index aaed234..b6b20fb 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -40,6 +40,7 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.concurrent.Refs;
public class Scrubber implements Closeable
{
@@ -142,7 +143,8 @@ public class Scrubber implements Closeable
public void scrub()
{
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline()))
+ try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline());
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
index aad479b..12586e5 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
@@ -123,11 +124,14 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
logger.trace("Index summaries for compacting SSTables are using {} MB of space",
(memoryPoolBytes - remainingBytes) / 1024.0 / 1024.0);
- List<SSTableReader> newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
-
- for (LifecycleTransaction txn : transactions.values())
- txn.finish();
+ List<SSTableReader> newSSTables;
+ try (Refs<SSTableReader> refs = Refs.ref(sstablesByHotness))
+ {
+ newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
+ for (LifecycleTransaction txn : transactions.values())
+ txn.finish();
+ }
total = 0;
for (SSTableReader sstable : Iterables.concat(compacting, oldFormatSSTables, newSSTables))
total += sstable.getIndexSummaryOffHeapSize();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[05/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d7329a63
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d7329a63
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d7329a63
Branch: refs/heads/cassandra-3.11
Commit: d7329a639f3761cacf360e1345c8bd2aef16a71b
Parents: a9225f9 3cd2c3c
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Dec 11 09:00:46 2017 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 09:00:46 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../cassandra/db/compaction/CompactionManager.java | 1 +
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a5a97db,c1e81fd..8cff716
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,81 -1,7 +1,83 @@@
-2.2.12
+3.0.16
+ * Fix SStable ordering by max timestamp in SinglePartitionReadCommand (CASSANDRA-14010)
+ * Accept role names containing forward-slash (CASSANDRA-14088)
+ * Optimize CRC check chance probability calculations (CASSANDRA-14094)
+ * Fix cleanup on keyspace with no replicas (CASSANDRA-13526)
+ * Fix updating base table rows with TTL not removing materialized view entries (CASSANDRA-14071)
+ * Reduce garbage created by DynamicSnitch (CASSANDRA-14091)
+ * More frequent commitlog chained markers (CASSANDRA-13987)
+ * Fix serialized size of DataLimits (CASSANDRA-14057)
+ * Add flag to allow dropping oversized read repair mutations (CASSANDRA-13975)
+ * Fix SSTableLoader logger message (CASSANDRA-14003)
+ * Fix repair race that caused gossip to block (CASSANDRA-13849)
+ * Tracing interferes with digest requests when using RandomPartitioner (CASSANDRA-13964)
+ * Add flag to disable materialized views, and warnings on creation (CASSANDRA-13959)
+ * Don't let user drop or generally break tables in system_distributed (CASSANDRA-13813)
+ * Provide a JMX call to sync schema with local storage (CASSANDRA-13954)
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
+ * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
++Merged from 2.2:
+ * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873)
-2.2.11
+
+3.0.15
+ * Improve TRUNCATE performance (CASSANDRA-13909)
+ * Implement short read protection on partition boundaries (CASSANDRA-13595)
+ * Fix ISE thrown by UPI.Serializer.hasNext() for some SELECT queries (CASSANDRA-13911)
+ * Filter header only commit logs before recovery (CASSANDRA-13918)
+ * AssertionError prepending to a list (CASSANDRA-13149)
+ * Fix support for SuperColumn tables (CASSANDRA-12373)
+ * Handle limit correctly on tables with strict liveness (CASSANDRA-13883)
+ * Fix missing original update in TriggerExecutor (CASSANDRA-13894)
+ * Remove non-rpc-ready nodes from counter leader candidates (CASSANDRA-13043)
+ * Improve short read protection performance (CASSANDRA-13794)
+ * Fix sstable reader to support range-tombstone-marker for multi-slices (CASSANDRA-13787)
+ * Fix short read protection for tables with no clustering columns (CASSANDRA-13880)
+ * Make isBuilt volatile in PartitionUpdate (CASSANDRA-13619)
+ * Prevent integer overflow of timestamps in CellTest and RowsTest (CASSANDRA-13866)
+ * Fix counter application order in short read protection (CASSANDRA-12872)
+ * Don't block RepairJob execution on validation futures (CASSANDRA-13797)
+ * Wait for all management tasks to complete before shutting down CLSM (CASSANDRA-13123)
+ * INSERT statement fails when Tuple type is used as clustering column with default DESC order (CASSANDRA-13717)
+ * Fix pending view mutations handling and cleanup batchlog when there are local and remote paired mutations (CASSANDRA-13069)
+ * Improve config validation and documentation on overflow and NPE (CASSANDRA-13622)
+ * Range deletes in a CAS batch are ignored (CASSANDRA-13655)
+ * Avoid assertion error when IndexSummary > 2G (CASSANDRA-12014)
+ * Change repair midpoint logging for tiny ranges (CASSANDRA-13603)
+ * Better handle corrupt final commitlog segment (CASSANDRA-11995)
+ * StreamingHistogram is not thread safe (CASSANDRA-13756)
+ * Fix MV timestamp issues (CASSANDRA-11500)
+ * Better tolerate improperly formatted bcrypt hashes (CASSANDRA-13626)
+ * Fix race condition in read command serialization (CASSANDRA-13363)
+ * Enable segement creation before recovering commitlogs (CASSANDRA-13587)
+ * Fix AssertionError in short read protection (CASSANDRA-13747)
+ * Don't skip corrupted sstables on startup (CASSANDRA-13620)
+ * Fix the merging of cells with different user type versions (CASSANDRA-13776)
+ * Copy session properties on cqlsh.py do_login (CASSANDRA-13640)
+ * Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719)
+ * Don't let stress write warmup data if n=0 (CASSANDRA-13773)
+ * Gossip thread slows down when using batch commit log (CASSANDRA-12966)
+ * Randomize batchlog endpoint selection with only 1 or 2 racks (CASSANDRA-12884)
+ * Fix digest calculation for counter cells (CASSANDRA-13750)
+ * Fix ColumnDefinition.cellValueType() for non-frozen collection and change SSTabledump to use type.toJSONString() (CASSANDRA-13573)
+ * Skip materialized view addition if the base table doesn't exist (CASSANDRA-13737)
+ * Drop table should remove corresponding entries in dropped_columns table (CASSANDRA-13730)
+ * Log warn message until legacy auth tables have been migrated (CASSANDRA-13371)
+ * Fix incorrect [2.1 <- 3.0] serialization of counter cells created in 2.0 (CASSANDRA-13691)
+ * Fix invalid writetime for null cells (CASSANDRA-13711)
+ * Fix ALTER TABLE statement to atomically propagate changes to the table and its MVs (CASSANDRA-12952)
+ * Fixed ambiguous output of nodetool tablestats command (CASSANDRA-13722)
+ * JMXEnabledThreadPoolExecutor with corePoolSize equal to maxPoolSize (Backport CASSANDRA-13329)
+ * Fix Digest mismatch Exception if hints file has UnknownColumnFamily (CASSANDRA-13696)
+ * Purge tombstones created by expired cells (CASSANDRA-13643)
+ * Make concat work with iterators that have different subsets of columns (CASSANDRA-13482)
+ * Set test.runners based on cores and memory size (CASSANDRA-13078)
+ * Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557)
+ * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606)
+ * Fix secondary index queries on COMPACT tables (CASSANDRA-13627)
+ * Nodetool listsnapshots output is missing a newline, if there are no snapshots (CASSANDRA-13568)
+ * sstabledump reports incorrect usage for argument order (CASSANDRA-13532)
+Merged from 2.2:
* Safely handle empty buffers when outputting to JSON (CASSANDRA-13868)
* Copy session properties on cqlsh.py do_login (CASSANDRA-13847)
* Fix load over calculated issue in IndexSummaryRedistribution (CASSANDRA-13738)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index fdda562,2e69b6f..eaeb089
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -869,16 -850,18 +869,17 @@@ public class CompactionManager implemen
if (compactionFileLocation == null)
throw new IOException("disk full");
- ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
- CleanupInfo ci = new CleanupInfo(sstable, scanner);
-
- metrics.beginCompaction(ci);
List<SSTableReader> finished;
- try (SSTableRewriter writer = new SSTableRewriter(cfs, txn, sstable.maxDataAge, false);
- CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs));
- Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
+ int nowInSec = FBUtilities.nowInSeconds();
+ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge, false);
+ ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
+ CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
++ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable));
+ CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics))
{
- writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));
+ writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, txn));
- while (scanner.hasNext())
+ while (ci.hasNext())
{
if (ci.isStopRequested())
throw new CompactionInterruptedException(ci.getCompactionInfo());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index c010891,b6b20fb..c8e0c53
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -35,7 -36,11 +35,8 @@@ import org.apache.cassandra.io.sstable.
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.OutputHandler;
-import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.*;
+ import org.apache.cassandra.utils.concurrent.Refs;
public class Scrubber implements Closeable
{
@@@ -135,17 -140,11 +136,18 @@@
this.nextRowPositionFromIndex = 0;
}
+ private UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
+ {
+ return checkData ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
+ }
+
public void scrub()
{
+ List<SSTableReader> finished = new ArrayList<>();
+ boolean completed = false;
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge, transaction.isOffline()))
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline());
++ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge, transaction.isOffline());
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[06/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d7329a63
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d7329a63
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d7329a63
Branch: refs/heads/cassandra-3.0
Commit: d7329a639f3761cacf360e1345c8bd2aef16a71b
Parents: a9225f9 3cd2c3c
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Dec 11 09:00:46 2017 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 09:00:46 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../cassandra/db/compaction/CompactionManager.java | 1 +
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a5a97db,c1e81fd..8cff716
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,81 -1,7 +1,83 @@@
-2.2.12
+3.0.16
+ * Fix SStable ordering by max timestamp in SinglePartitionReadCommand (CASSANDRA-14010)
+ * Accept role names containing forward-slash (CASSANDRA-14088)
+ * Optimize CRC check chance probability calculations (CASSANDRA-14094)
+ * Fix cleanup on keyspace with no replicas (CASSANDRA-13526)
+ * Fix updating base table rows with TTL not removing materialized view entries (CASSANDRA-14071)
+ * Reduce garbage created by DynamicSnitch (CASSANDRA-14091)
+ * More frequent commitlog chained markers (CASSANDRA-13987)
+ * Fix serialized size of DataLimits (CASSANDRA-14057)
+ * Add flag to allow dropping oversized read repair mutations (CASSANDRA-13975)
+ * Fix SSTableLoader logger message (CASSANDRA-14003)
+ * Fix repair race that caused gossip to block (CASSANDRA-13849)
+ * Tracing interferes with digest requests when using RandomPartitioner (CASSANDRA-13964)
+ * Add flag to disable materialized views, and warnings on creation (CASSANDRA-13959)
+ * Don't let user drop or generally break tables in system_distributed (CASSANDRA-13813)
+ * Provide a JMX call to sync schema with local storage (CASSANDRA-13954)
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
+ * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
++Merged from 2.2:
+ * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873)
-2.2.11
+
+3.0.15
+ * Improve TRUNCATE performance (CASSANDRA-13909)
+ * Implement short read protection on partition boundaries (CASSANDRA-13595)
+ * Fix ISE thrown by UPI.Serializer.hasNext() for some SELECT queries (CASSANDRA-13911)
+ * Filter header only commit logs before recovery (CASSANDRA-13918)
+ * AssertionError prepending to a list (CASSANDRA-13149)
+ * Fix support for SuperColumn tables (CASSANDRA-12373)
+ * Handle limit correctly on tables with strict liveness (CASSANDRA-13883)
+ * Fix missing original update in TriggerExecutor (CASSANDRA-13894)
+ * Remove non-rpc-ready nodes from counter leader candidates (CASSANDRA-13043)
+ * Improve short read protection performance (CASSANDRA-13794)
+ * Fix sstable reader to support range-tombstone-marker for multi-slices (CASSANDRA-13787)
+ * Fix short read protection for tables with no clustering columns (CASSANDRA-13880)
+ * Make isBuilt volatile in PartitionUpdate (CASSANDRA-13619)
+ * Prevent integer overflow of timestamps in CellTest and RowsTest (CASSANDRA-13866)
+ * Fix counter application order in short read protection (CASSANDRA-12872)
+ * Don't block RepairJob execution on validation futures (CASSANDRA-13797)
+ * Wait for all management tasks to complete before shutting down CLSM (CASSANDRA-13123)
+ * INSERT statement fails when Tuple type is used as clustering column with default DESC order (CASSANDRA-13717)
+ * Fix pending view mutations handling and cleanup batchlog when there are local and remote paired mutations (CASSANDRA-13069)
+ * Improve config validation and documentation on overflow and NPE (CASSANDRA-13622)
+ * Range deletes in a CAS batch are ignored (CASSANDRA-13655)
+ * Avoid assertion error when IndexSummary > 2G (CASSANDRA-12014)
+ * Change repair midpoint logging for tiny ranges (CASSANDRA-13603)
+ * Better handle corrupt final commitlog segment (CASSANDRA-11995)
+ * StreamingHistogram is not thread safe (CASSANDRA-13756)
+ * Fix MV timestamp issues (CASSANDRA-11500)
+ * Better tolerate improperly formatted bcrypt hashes (CASSANDRA-13626)
+ * Fix race condition in read command serialization (CASSANDRA-13363)
+ * Enable segement creation before recovering commitlogs (CASSANDRA-13587)
+ * Fix AssertionError in short read protection (CASSANDRA-13747)
+ * Don't skip corrupted sstables on startup (CASSANDRA-13620)
+ * Fix the merging of cells with different user type versions (CASSANDRA-13776)
+ * Copy session properties on cqlsh.py do_login (CASSANDRA-13640)
+ * Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719)
+ * Don't let stress write warmup data if n=0 (CASSANDRA-13773)
+ * Gossip thread slows down when using batch commit log (CASSANDRA-12966)
+ * Randomize batchlog endpoint selection with only 1 or 2 racks (CASSANDRA-12884)
+ * Fix digest calculation for counter cells (CASSANDRA-13750)
+ * Fix ColumnDefinition.cellValueType() for non-frozen collection and change SSTabledump to use type.toJSONString() (CASSANDRA-13573)
+ * Skip materialized view addition if the base table doesn't exist (CASSANDRA-13737)
+ * Drop table should remove corresponding entries in dropped_columns table (CASSANDRA-13730)
+ * Log warn message until legacy auth tables have been migrated (CASSANDRA-13371)
+ * Fix incorrect [2.1 <- 3.0] serialization of counter cells created in 2.0 (CASSANDRA-13691)
+ * Fix invalid writetime for null cells (CASSANDRA-13711)
+ * Fix ALTER TABLE statement to atomically propagate changes to the table and its MVs (CASSANDRA-12952)
+ * Fixed ambiguous output of nodetool tablestats command (CASSANDRA-13722)
+ * JMXEnabledThreadPoolExecutor with corePoolSize equal to maxPoolSize (Backport CASSANDRA-13329)
+ * Fix Digest mismatch Exception if hints file has UnknownColumnFamily (CASSANDRA-13696)
+ * Purge tombstones created by expired cells (CASSANDRA-13643)
+ * Make concat work with iterators that have different subsets of columns (CASSANDRA-13482)
+ * Set test.runners based on cores and memory size (CASSANDRA-13078)
+ * Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557)
+ * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606)
+ * Fix secondary index queries on COMPACT tables (CASSANDRA-13627)
+ * Nodetool listsnapshots output is missing a newline, if there are no snapshots (CASSANDRA-13568)
+ * sstabledump reports incorrect usage for argument order (CASSANDRA-13532)
+Merged from 2.2:
* Safely handle empty buffers when outputting to JSON (CASSANDRA-13868)
* Copy session properties on cqlsh.py do_login (CASSANDRA-13847)
* Fix load over calculated issue in IndexSummaryRedistribution (CASSANDRA-13738)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index fdda562,2e69b6f..eaeb089
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -869,16 -850,18 +869,17 @@@ public class CompactionManager implemen
if (compactionFileLocation == null)
throw new IOException("disk full");
- ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
- CleanupInfo ci = new CleanupInfo(sstable, scanner);
-
- metrics.beginCompaction(ci);
List<SSTableReader> finished;
- try (SSTableRewriter writer = new SSTableRewriter(cfs, txn, sstable.maxDataAge, false);
- CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs));
- Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
+ int nowInSec = FBUtilities.nowInSeconds();
+ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge, false);
+ ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
+ CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
++ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable));
+ CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics))
{
- writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));
+ writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, txn));
- while (scanner.hasNext())
+ while (ci.hasNext())
{
if (ci.isStopRequested())
throw new CompactionInterruptedException(ci.getCompactionInfo());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index c010891,b6b20fb..c8e0c53
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -35,7 -36,11 +35,8 @@@ import org.apache.cassandra.io.sstable.
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.OutputHandler;
-import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.*;
+ import org.apache.cassandra.utils.concurrent.Refs;
public class Scrubber implements Closeable
{
@@@ -135,17 -140,11 +136,18 @@@
this.nextRowPositionFromIndex = 0;
}
+ private UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
+ {
+ return checkData ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
+ }
+
public void scrub()
{
+ List<SSTableReader> finished = new ArrayList<>();
+ boolean completed = false;
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge, transaction.isOffline()))
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline());
++ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge, transaction.isOffline());
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[10/10] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-3.11' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1cb05092
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1cb05092
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1cb05092
Branch: refs/heads/trunk
Commit: 1cb050922c7cc181e7c4e90fa2a52c23a87d5fc0
Parents: 7815014 817f3c2
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Dec 11 09:04:46 2017 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 09:04:46 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../cassandra/db/compaction/CompactionManager.java | 1 +
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1cb05092/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1cb05092/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index e5c8269,638e74c..c57b37a
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -1094,14 -1095,15 +1094,15 @@@ public class CompactionManager implemen
int nowInSec = FBUtilities.nowInSeconds();
try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge);
- ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, null);
+ ISSTableScanner scanner = cleanupStrategy.getScanner(sstable);
CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable));
CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics))
{
- writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, txn));
+ StatsMetadata metadata = sstable.getSSTableMetadata();
+ writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, sstable, txn));
long lastBytesScanned = 0;
-
while (ci.hasNext())
{
if (ci.isStopRequested())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1cb05092/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1cb05092/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
index 47c3fca,73626d3..24de918
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
@@@ -39,9 -40,9 +39,10 @@@ import org.apache.cassandra.db.compacti
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
+ import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
@@@ -112,13 -125,16 +113,16 @@@ public class IndexSummaryRedistributio
logger.trace("Index summaries for compacting SSTables are using {} MB of space",
(memoryPoolBytes - remainingBytes) / 1024.0 / 1024.0);
- List<SSTableReader> newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
-
- for (LifecycleTransaction txn : transactions.values())
- txn.finish();
+ List<SSTableReader> newSSTables;
+ try (Refs<SSTableReader> refs = Refs.ref(sstablesByHotness))
+ {
+ newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
+ for (LifecycleTransaction txn : transactions.values())
+ txn.finish();
+ }
total = 0;
- for (SSTableReader sstable : Iterables.concat(compacting, oldFormatSSTables, newSSTables))
+ for (SSTableReader sstable : Iterables.concat(compacting, newSSTables))
total += sstable.getIndexSummaryOffHeapSize();
logger.trace("Completed resizing of index summaries; current approximate memory used: {}",
FBUtilities.prettyPrintMemory(total));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[03/10] cassandra git commit: Grab refs during scrub,
index summary redistribution and cleanup
Posted by ma...@apache.org.
Grab refs during scrub, index summary redistribution and cleanup
Patch by marcuse; reviewed by Joel Knighton for CASSANDRA-13873
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3cd2c3c4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3cd2c3c4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3cd2c3c4
Branch: refs/heads/cassandra-3.11
Commit: 3cd2c3c4ea4286562b2cb8443d6173ee251e6212
Parents: 797de4a
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Oct 23 09:43:44 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 08:53:44 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +-
.../cassandra/db/compaction/CompactionManager.java | 3 ++-
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 752cbdc..c1e81fd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,5 @@
2.2.12
- *
+ * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873)
2.2.11
* Safely handle empty buffers when outputting to JSON (CASSANDRA-13868)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index cd50646..2e69b6f 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -856,7 +856,8 @@ public class CompactionManager implements CompactionManagerMBean
metrics.beginCompaction(ci);
List<SSTableReader> finished;
try (SSTableRewriter writer = new SSTableRewriter(cfs, txn, sstable.maxDataAge, false);
- CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs)))
+ CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs));
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index aaed234..b6b20fb 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -40,6 +40,7 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.concurrent.Refs;
public class Scrubber implements Closeable
{
@@ -142,7 +143,8 @@ public class Scrubber implements Closeable
public void scrub()
{
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline()))
+ try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline());
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
index aad479b..12586e5 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
@@ -123,11 +124,14 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
logger.trace("Index summaries for compacting SSTables are using {} MB of space",
(memoryPoolBytes - remainingBytes) / 1024.0 / 1024.0);
- List<SSTableReader> newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
-
- for (LifecycleTransaction txn : transactions.values())
- txn.finish();
+ List<SSTableReader> newSSTables;
+ try (Refs<SSTableReader> refs = Refs.ref(sstablesByHotness))
+ {
+ newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
+ for (LifecycleTransaction txn : transactions.values())
+ txn.finish();
+ }
total = 0;
for (SSTableReader sstable : Iterables.concat(compacting, oldFormatSSTables, newSSTables))
total += sstable.getIndexSummaryOffHeapSize();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[04/10] cassandra git commit: Grab refs during scrub,
index summary redistribution and cleanup
Posted by ma...@apache.org.
Grab refs during scrub, index summary redistribution and cleanup
Patch by marcuse; reviewed by Joel Knighton for CASSANDRA-13873
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3cd2c3c4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3cd2c3c4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3cd2c3c4
Branch: refs/heads/trunk
Commit: 3cd2c3c4ea4286562b2cb8443d6173ee251e6212
Parents: 797de4a
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Oct 23 09:43:44 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 08:53:44 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +-
.../cassandra/db/compaction/CompactionManager.java | 3 ++-
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 752cbdc..c1e81fd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,5 @@
2.2.12
- *
+ * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873)
2.2.11
* Safely handle empty buffers when outputting to JSON (CASSANDRA-13868)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index cd50646..2e69b6f 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -856,7 +856,8 @@ public class CompactionManager implements CompactionManagerMBean
metrics.beginCompaction(ci);
List<SSTableReader> finished;
try (SSTableRewriter writer = new SSTableRewriter(cfs, txn, sstable.maxDataAge, false);
- CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs)))
+ CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs));
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index aaed234..b6b20fb 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -40,6 +40,7 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.concurrent.Refs;
public class Scrubber implements Closeable
{
@@ -142,7 +143,8 @@ public class Scrubber implements Closeable
public void scrub()
{
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline()))
+ try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline());
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd2c3c4/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
index aad479b..12586e5 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
@@ -123,11 +124,14 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
logger.trace("Index summaries for compacting SSTables are using {} MB of space",
(memoryPoolBytes - remainingBytes) / 1024.0 / 1024.0);
- List<SSTableReader> newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
-
- for (LifecycleTransaction txn : transactions.values())
- txn.finish();
+ List<SSTableReader> newSSTables;
+ try (Refs<SSTableReader> refs = Refs.ref(sstablesByHotness))
+ {
+ newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes);
+ for (LifecycleTransaction txn : transactions.values())
+ txn.finish();
+ }
total = 0;
for (SSTableReader sstable : Iterables.concat(compacting, oldFormatSSTables, newSSTables))
total += sstable.getIndexSummaryOffHeapSize();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[07/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d7329a63
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d7329a63
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d7329a63
Branch: refs/heads/trunk
Commit: d7329a639f3761cacf360e1345c8bd2aef16a71b
Parents: a9225f9 3cd2c3c
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Dec 11 09:00:46 2017 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Dec 11 09:00:46 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../cassandra/db/compaction/CompactionManager.java | 1 +
.../org/apache/cassandra/db/compaction/Scrubber.java | 4 +++-
.../io/sstable/IndexSummaryRedistribution.java | 12 ++++++++----
4 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a5a97db,c1e81fd..8cff716
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,81 -1,7 +1,83 @@@
-2.2.12
+3.0.16
+ * Fix SStable ordering by max timestamp in SinglePartitionReadCommand (CASSANDRA-14010)
+ * Accept role names containing forward-slash (CASSANDRA-14088)
+ * Optimize CRC check chance probability calculations (CASSANDRA-14094)
+ * Fix cleanup on keyspace with no replicas (CASSANDRA-13526)
+ * Fix updating base table rows with TTL not removing materialized view entries (CASSANDRA-14071)
+ * Reduce garbage created by DynamicSnitch (CASSANDRA-14091)
+ * More frequent commitlog chained markers (CASSANDRA-13987)
+ * Fix serialized size of DataLimits (CASSANDRA-14057)
+ * Add flag to allow dropping oversized read repair mutations (CASSANDRA-13975)
+ * Fix SSTableLoader logger message (CASSANDRA-14003)
+ * Fix repair race that caused gossip to block (CASSANDRA-13849)
+ * Tracing interferes with digest requests when using RandomPartitioner (CASSANDRA-13964)
+ * Add flag to disable materialized views, and warnings on creation (CASSANDRA-13959)
+ * Don't let user drop or generally break tables in system_distributed (CASSANDRA-13813)
+ * Provide a JMX call to sync schema with local storage (CASSANDRA-13954)
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
+ * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
++Merged from 2.2:
+ * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873)
-2.2.11
+
+3.0.15
+ * Improve TRUNCATE performance (CASSANDRA-13909)
+ * Implement short read protection on partition boundaries (CASSANDRA-13595)
+ * Fix ISE thrown by UPI.Serializer.hasNext() for some SELECT queries (CASSANDRA-13911)
+ * Filter header only commit logs before recovery (CASSANDRA-13918)
+ * AssertionError prepending to a list (CASSANDRA-13149)
+ * Fix support for SuperColumn tables (CASSANDRA-12373)
+ * Handle limit correctly on tables with strict liveness (CASSANDRA-13883)
+ * Fix missing original update in TriggerExecutor (CASSANDRA-13894)
+ * Remove non-rpc-ready nodes from counter leader candidates (CASSANDRA-13043)
+ * Improve short read protection performance (CASSANDRA-13794)
+ * Fix sstable reader to support range-tombstone-marker for multi-slices (CASSANDRA-13787)
+ * Fix short read protection for tables with no clustering columns (CASSANDRA-13880)
+ * Make isBuilt volatile in PartitionUpdate (CASSANDRA-13619)
+ * Prevent integer overflow of timestamps in CellTest and RowsTest (CASSANDRA-13866)
+ * Fix counter application order in short read protection (CASSANDRA-12872)
+ * Don't block RepairJob execution on validation futures (CASSANDRA-13797)
+ * Wait for all management tasks to complete before shutting down CLSM (CASSANDRA-13123)
+ * INSERT statement fails when Tuple type is used as clustering column with default DESC order (CASSANDRA-13717)
+ * Fix pending view mutations handling and cleanup batchlog when there are local and remote paired mutations (CASSANDRA-13069)
+ * Improve config validation and documentation on overflow and NPE (CASSANDRA-13622)
+ * Range deletes in a CAS batch are ignored (CASSANDRA-13655)
+ * Avoid assertion error when IndexSummary > 2G (CASSANDRA-12014)
+ * Change repair midpoint logging for tiny ranges (CASSANDRA-13603)
+ * Better handle corrupt final commitlog segment (CASSANDRA-11995)
+ * StreamingHistogram is not thread safe (CASSANDRA-13756)
+ * Fix MV timestamp issues (CASSANDRA-11500)
+ * Better tolerate improperly formatted bcrypt hashes (CASSANDRA-13626)
+ * Fix race condition in read command serialization (CASSANDRA-13363)
+ * Enable segement creation before recovering commitlogs (CASSANDRA-13587)
+ * Fix AssertionError in short read protection (CASSANDRA-13747)
+ * Don't skip corrupted sstables on startup (CASSANDRA-13620)
+ * Fix the merging of cells with different user type versions (CASSANDRA-13776)
+ * Copy session properties on cqlsh.py do_login (CASSANDRA-13640)
+ * Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719)
+ * Don't let stress write warmup data if n=0 (CASSANDRA-13773)
+ * Gossip thread slows down when using batch commit log (CASSANDRA-12966)
+ * Randomize batchlog endpoint selection with only 1 or 2 racks (CASSANDRA-12884)
+ * Fix digest calculation for counter cells (CASSANDRA-13750)
+ * Fix ColumnDefinition.cellValueType() for non-frozen collection and change SSTabledump to use type.toJSONString() (CASSANDRA-13573)
+ * Skip materialized view addition if the base table doesn't exist (CASSANDRA-13737)
+ * Drop table should remove corresponding entries in dropped_columns table (CASSANDRA-13730)
+ * Log warn message until legacy auth tables have been migrated (CASSANDRA-13371)
+ * Fix incorrect [2.1 <- 3.0] serialization of counter cells created in 2.0 (CASSANDRA-13691)
+ * Fix invalid writetime for null cells (CASSANDRA-13711)
+ * Fix ALTER TABLE statement to atomically propagate changes to the table and its MVs (CASSANDRA-12952)
+ * Fixed ambiguous output of nodetool tablestats command (CASSANDRA-13722)
+ * JMXEnabledThreadPoolExecutor with corePoolSize equal to maxPoolSize (Backport CASSANDRA-13329)
+ * Fix Digest mismatch Exception if hints file has UnknownColumnFamily (CASSANDRA-13696)
+ * Purge tombstones created by expired cells (CASSANDRA-13643)
+ * Make concat work with iterators that have different subsets of columns (CASSANDRA-13482)
+ * Set test.runners based on cores and memory size (CASSANDRA-13078)
+ * Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557)
+ * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606)
+ * Fix secondary index queries on COMPACT tables (CASSANDRA-13627)
+ * Nodetool listsnapshots output is missing a newline, if there are no snapshots (CASSANDRA-13568)
+ * sstabledump reports incorrect usage for argument order (CASSANDRA-13532)
+Merged from 2.2:
* Safely handle empty buffers when outputting to JSON (CASSANDRA-13868)
* Copy session properties on cqlsh.py do_login (CASSANDRA-13847)
* Fix load over calculated issue in IndexSummaryRedistribution (CASSANDRA-13738)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index fdda562,2e69b6f..eaeb089
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -869,16 -850,18 +869,17 @@@ public class CompactionManager implemen
if (compactionFileLocation == null)
throw new IOException("disk full");
- ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
- CleanupInfo ci = new CleanupInfo(sstable, scanner);
-
- metrics.beginCompaction(ci);
List<SSTableReader> finished;
- try (SSTableRewriter writer = new SSTableRewriter(cfs, txn, sstable.maxDataAge, false);
- CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs));
- Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
+ int nowInSec = FBUtilities.nowInSeconds();
+ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge, false);
+ ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
+ CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
++ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable));
+ CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics))
{
- writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));
+ writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, txn));
- while (scanner.hasNext())
+ while (ci.hasNext())
{
if (ci.isStopRequested())
throw new CompactionInterruptedException(ci.getCompactionInfo());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index c010891,b6b20fb..c8e0c53
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -35,7 -36,11 +35,8 @@@ import org.apache.cassandra.io.sstable.
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.OutputHandler;
-import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.*;
+ import org.apache.cassandra.utils.concurrent.Refs;
public class Scrubber implements Closeable
{
@@@ -135,17 -140,11 +136,18 @@@
this.nextRowPositionFromIndex = 0;
}
+ private UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
+ {
+ return checkData ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
+ }
+
public void scrub()
{
+ List<SSTableReader> finished = new ArrayList<>();
+ boolean completed = false;
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge, transaction.isOffline()))
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline());
++ try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge, transaction.isOffline());
+ Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d7329a63/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org