You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2016/02/11 08:30:29 UTC
[01/10] cassandra git commit: Properly release sstable reference when
doing offline scrub
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 6c6b7e40c -> 96549946f
refs/heads/cassandra-2.2 d7195060c -> 7d7ff7f3c
refs/heads/cassandra-3.0 89c558a43 -> d4364075d
refs/heads/trunk 7863bd3bf -> 5c2a4ef28
Properly release sstable reference when doing offline scrub
Patch by marcuse; reviewed by Stefania for CASSANDRA-10697
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/96549946
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/96549946
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/96549946
Branch: refs/heads/cassandra-2.1
Commit: 96549946fe994ea8311e72e5bf6f51c2124f7bb4
Parents: 6c6b7e4
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Feb 9 09:53:59 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:20:47 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/compaction/Scrubber.java | 4 ++++
.../apache/cassandra/tools/StandaloneScrubber.java | 5 ++++-
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 63 -> 75 bytes
.../Keyspace1-StandardInteger1-ka-2-Summary.db | Bin 80 -> 112 bytes
5 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3bbef11..9f51291 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.14
+ * Properly release sstable ref when doing offline scrub (CASSANDRA-10697)
* Improve nodetool status performance for large cluster (CASSANDRA-7238)
* Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* Gossiper#isEnabled is not thread safe (CASSANDRA-11116)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index e02f901..8bfd37b 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -300,6 +300,8 @@ public class Scrubber implements Closeable
newInOrderSstable = inOrderWriter.closeAndOpenReader(sstable.maxDataAge);
if (!isOffline)
cfs.getDataTracker().addSSTables(Collections.singleton(newInOrderSstable));
+ else if (newInOrderSstable != null)
+ newInOrderSstable.selfRef().release();
outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrderRows.size(), sstable, newInOrderSstable));
}
@@ -318,6 +320,8 @@ public class Scrubber implements Closeable
finally
{
controller.close();
+ if (isOffline && newSstable != null)
+ newSstable.selfRef().release();
}
if (newSstable == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 79ba0bd..fdf6c8d 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -122,13 +122,16 @@ public class StandaloneScrubber
// Remove the sstable (it's been copied by scrub and snapshotted)
sstable.markObsolete(null);
- sstable.selfRef().release();
}
catch (Exception e)
{
System.err.println(String.format("Error scrubbing %s: %s", sstable, e.getMessage()));
e.printStackTrace(System.err);
}
+ finally
+ {
+ sstable.selfRef().release();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
index 7621f07..6eb7650 100644
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db b/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db
index 22cfa6a..190922a 100644
Binary files a/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db and b/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db differ
[04/10] cassandra git commit: Properly release sstable reference when
doing offline scrub
Posted by ma...@apache.org.
Properly release sstable reference when doing offline scrub
Patch by marcuse; reviewed by Stefania for CASSANDRA-10697
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/96549946
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/96549946
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/96549946
Branch: refs/heads/trunk
Commit: 96549946fe994ea8311e72e5bf6f51c2124f7bb4
Parents: 6c6b7e4
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Feb 9 09:53:59 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:20:47 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/compaction/Scrubber.java | 4 ++++
.../apache/cassandra/tools/StandaloneScrubber.java | 5 ++++-
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 63 -> 75 bytes
.../Keyspace1-StandardInteger1-ka-2-Summary.db | Bin 80 -> 112 bytes
5 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3bbef11..9f51291 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.14
+ * Properly release sstable ref when doing offline scrub (CASSANDRA-10697)
* Improve nodetool status performance for large cluster (CASSANDRA-7238)
* Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* Gossiper#isEnabled is not thread safe (CASSANDRA-11116)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index e02f901..8bfd37b 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -300,6 +300,8 @@ public class Scrubber implements Closeable
newInOrderSstable = inOrderWriter.closeAndOpenReader(sstable.maxDataAge);
if (!isOffline)
cfs.getDataTracker().addSSTables(Collections.singleton(newInOrderSstable));
+ else if (newInOrderSstable != null)
+ newInOrderSstable.selfRef().release();
outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrderRows.size(), sstable, newInOrderSstable));
}
@@ -318,6 +320,8 @@ public class Scrubber implements Closeable
finally
{
controller.close();
+ if (isOffline && newSstable != null)
+ newSstable.selfRef().release();
}
if (newSstable == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 79ba0bd..fdf6c8d 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -122,13 +122,16 @@ public class StandaloneScrubber
// Remove the sstable (it's been copied by scrub and snapshotted)
sstable.markObsolete(null);
- sstable.selfRef().release();
}
catch (Exception e)
{
System.err.println(String.format("Error scrubbing %s: %s", sstable, e.getMessage()));
e.printStackTrace(System.err);
}
+ finally
+ {
+ sstable.selfRef().release();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
index 7621f07..6eb7650 100644
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db b/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db
index 22cfa6a..190922a 100644
Binary files a/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db and b/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db differ
[06/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d7ff7f3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d7ff7f3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d7ff7f3
Branch: refs/heads/cassandra-2.2
Commit: 7d7ff7f3cd317531a096e9f06c0bac9f5e139496
Parents: d719506 9654994
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Feb 11 08:25:14 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:25:14 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 12 +++---------
.../cassandra/db/compaction/Scrubber.java | 19 ++++++++++---------
.../cassandra/tools/StandaloneScrubber.java | 2 +-
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 63 -> 75 bytes
.../unit/org/apache/cassandra/db/ScrubTest.java | 14 +++++++-------
6 files changed, 22 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 318672f,9f51291..7565386
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,11 -1,9 +1,12 @@@
-2.1.14
+2.2.6
+ * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
+ * Fix paging on DISTINCT queries repeats result when first row in partition changes (CASSANDRA-10010)
+Merged from 2.1:
+ * Properly release sstable ref when doing offline scrub (CASSANDRA-10697)
* Improve nodetool status performance for large cluster (CASSANDRA-7238)
- * Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* Gossiper#isEnabled is not thread safe (CASSANDRA-11116)
* Avoid major compaction mixing repaired and unrepaired sstables in DTCS (CASSANDRA-11113)
+ * Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* test_bulk_round_trip_blogposts is failing occasionally (CASSANDRA-10938)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index c51ed7d,55b873a..8ca9852
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -327,15 -340,9 +327,9 @@@ public class CompactionManager implemen
}
}
- public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData) throws InterruptedException, ExecutionException
+ public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData)
+ throws InterruptedException, ExecutionException
{
- return performScrub(cfs, skipCorrupted, checkData, false);
- }
-
- public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData, final boolean offline)
- throws InterruptedException, ExecutionException
- {
- assert !cfs.isIndex();
return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
{
@Override
@@@ -345,30 -352,11 +339,30 @@@
}
@Override
- public void execute(SSTableReader input) throws IOException
+ public void execute(LifecycleTransaction input) throws IOException
{
- scrubOne(cfs, input, skipCorrupted, checkData, offline);
+ scrubOne(cfs, input, skipCorrupted, checkData);
}
- });
+ }, OperationType.SCRUB);
+ }
+
+ public AllSSTableOpStatus performVerify(final ColumnFamilyStore cfs, final boolean extendedVerify) throws InterruptedException, ExecutionException
+ {
+ assert !cfs.isIndex();
+ return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
+ {
+ @Override
+ public Iterable<SSTableReader> filterSSTables(LifecycleTransaction input)
+ {
+ return input.originals();
+ }
+
+ @Override
+ public void execute(LifecycleTransaction input) throws IOException
+ {
+ verifyOne(cfs, input.onlyOne(), extendedVerify);
+ }
+ }, OperationType.VERIFY);
}
public AllSSTableOpStatus performSSTableRewrite(final ColumnFamilyStore cfs, final boolean excludeCurrentVersion) throws InterruptedException, ExecutionException
@@@ -705,14 -683,14 +699,14 @@@
}
}
- private void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData, boolean offline) throws IOException
- private void scrubOne(ColumnFamilyStore cfs, SSTableReader sstable, boolean skipCorrupted, boolean checkData) throws IOException
++ private void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData) throws IOException
{
- Scrubber scrubber = new Scrubber(cfs, sstable, skipCorrupted, false, checkData);
+ CompactionInfo.Holder scrubInfo = null;
- try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, offline, checkData))
- CompactionInfo.Holder scrubInfo = scrubber.getScrubInfo();
- metrics.beginCompaction(scrubInfo);
- try
++ try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, checkData))
{
+ scrubInfo = scrubber.getScrubInfo();
+ metrics.beginCompaction(scrubInfo);
scrubber.scrub();
}
finally
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 9fd8560,8bfd37b..e9137e2
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -54,10 -49,9 +54,8 @@@ public class Scrubber implements Closea
private final RandomAccessReader dataFile;
private final RandomAccessReader indexFile;
private final ScrubInfo scrubInfo;
-
- private final boolean isOffline;
+ private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
- private final boolean isOffline;
-
private SSTableReader newSstable;
private SSTableReader newInOrderSstable;
@@@ -81,21 -75,19 +79,20 @@@
};
private final SortedSet<Row> outOfOrderRows = new TreeSet<>(rowComparator);
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException
- public Scrubber(ColumnFamilyStore cfs, SSTableReader sstable, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException
++ public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData) throws IOException
{
- this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
- this(cfs, sstable, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
++ this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), checkData);
}
- public Scrubber(ColumnFamilyStore cfs, SSTableReader sstable, boolean skipCorrupted, OutputHandler outputHandler, boolean isOffline, boolean checkData) throws IOException
+ @SuppressWarnings("resource")
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, OutputHandler outputHandler, boolean isOffline, boolean checkData) throws IOException
++ public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, OutputHandler outputHandler, boolean checkData) throws IOException
{
this.cfs = cfs;
- this.sstable = sstable;
+ this.transaction = transaction;
+ this.sstable = transaction.onlyOne();
this.outputHandler = outputHandler;
this.skipCorrupted = skipCorrupted;
-- this.isOffline = isOffline;
- this.validateColumns = checkData;
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata);
List<SSTableReader> toScrub = Collections.singletonList(sstable);
@@@ -105,7 -97,7 +102,7 @@@
throw new IOException("disk full");
// If we run scrub offline, we should never purge tombstone, as we cannot know if other sstable have data that the tombstone deletes.
-- this.controller = isOffline
++ this.controller = transaction.isOffline()
? new ScrubController(cfs)
: new CompactionController(cfs, Collections.singleton(sstable), CompactionManager.getDefaultGcBefore(cfs));
this.isCommutative = cfs.metadata.isCounter();
@@@ -126,7 -117,7 +123,7 @@@
// we'll also loop through the index at the same time, using the position from the index to recover if the
// row header (key or data size) is corrupt. (This means our position in the index file will be one row
// "ahead" of the data file.)
-- this.dataFile = isOffline
++ this.dataFile = transaction.isOffline()
? sstable.openDataReader()
: sstable.openDataReader(CompactionManager.instance.getRateLimiter());
@@@ -143,7 -134,9 +140,7 @@@
public void scrub()
{
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, isOffline))
- Set<SSTableReader> oldSSTable = Sets.newHashSet(sstable);
- SSTableRewriter writer = new SSTableRewriter(cfs, oldSSTable, sstable.maxDataAge, isOffline);
- try
++ try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline()))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
@@@ -299,13 -294,14 +296,15 @@@
{
// out of order rows, but no bad rows found - we can keep our repairedAt time
long repairedAt = badRows > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt;
- SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable);
- for (Row row : outOfOrderRows)
- inOrderWriter.append(row.key, row.cf);
- newInOrderSstable = inOrderWriter.closeAndOpenReader(sstable.maxDataAge);
- if (!isOffline)
- cfs.getDataTracker().addSSTables(Collections.singleton(newInOrderSstable));
- else if (newInOrderSstable != null)
+ try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable);)
+ {
+ for (Row row : outOfOrderRows)
+ inOrderWriter.append(row.key, row.cf);
+ newInOrderSstable = inOrderWriter.finish(-1, sstable.maxDataAge, true);
+ }
+ transaction.update(newInOrderSstable, false);
++ if (transaction.isOffline() && newInOrderSstable != null)
+ newInOrderSstable.selfRef().release();
outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrderRows.size(), sstable, newInOrderSstable));
}
@@@ -321,6 -320,8 +320,8 @@@
finally
{
controller.close();
- if (isOffline && newSstable != null)
++ if (transaction.isOffline() && newSstable != null)
+ newSstable.selfRef().release();
}
if (newSstable == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 3551b3d,fdf6c8d..a486a13
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@@ -119,10 -108,10 +119,10 @@@ public class StandaloneScrubbe
{
for (SSTableReader sstable : sstables)
{
- try
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable))
{
- Scrubber scrubber = new Scrubber(cfs, sstable, options.skipCorrupted, handler, true, !options.noValidate);
- try
+ txn.obsoleteOriginals(); // make sure originals are deleted and avoid NPE if index is missing, CASSANDRA-9591
- try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, true, !options.noValidate))
++ try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, !options.noValidate))
{
scrubber.scrub();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/ScrubTest.java
index b69a1f8,167671b..c0cde40
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@@ -166,21 -130,19 +166,21 @@@ public class ScrubTes
overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
// with skipCorrupted == false, the scrub is expected to fail
- Scrubber scrubber = new Scrubber(cfs, sstable, false, false, true);
- try
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, false, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
{
scrubber.scrub();
fail("Expected a CorruptSSTableException to be thrown");
}
catch (IOError err) {}
- // with skipCorrupted == true, the corrupt row will be skipped
+ // with skipCorrupted == true, the corrupt rows will be skipped
Scrubber.ScrubResult scrubResult;
- scrubber = new Scrubber(cfs, sstable, true, false, true);
- scrubResult = scrubber.scrubWithResult();
- scrubber.close();
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, false, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, true, true);)
+ {
+ scrubResult = scrubber.scrubWithResult();
+ }
assertNotNull(scrubResult);
@@@ -226,24 -188,20 +226,24 @@@
overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
// with skipCorrupted == false, the scrub is expected to fail
- Scrubber scrubber = new Scrubber(cfs, sstable, false, false, true);
- try
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, false, true))
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true))
{
+ // with skipCorrupted == true, the corrupt row will be skipped
scrubber.scrub();
fail("Expected a CorruptSSTableException to be thrown");
}
catch (IOError err) {}
- // with skipCorrupted == true, the corrupt row will be skipped
- scrubber = new Scrubber(cfs, sstable, true, false, true);
- scrubber.scrub();
- scrubber.close();
- assertEquals(1, cfs.getSSTables().size());
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, false, true))
++ Scrubber scrubber = new Scrubber(cfs, txn, true, true))
+ {
+ // with skipCorrupted == true, the corrupt row will be skipped
+ scrubber.scrub();
+ scrubber.close();
+ }
+ assertEquals(1, cfs.getSSTables().size());
// verify that we can read all of the rows, and there is now one less row
rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
assertEquals(1, rows.size());
@@@ -411,16 -353,11 +411,16 @@@
components.add(Component.STATS);
components.add(Component.SUMMARY);
components.add(Component.TOC);
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata);
- Scrubber scrubber = new Scrubber(cfs, sstable, false, true, true);
- scrubber.scrub();
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
+ if (sstable.last.compareTo(sstable.first) < 0)
+ sstable.last = sstable.first;
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
+ {
+ scrubber.scrub();
+ }
cfs.loadNewSSTables();
List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
assert isRowOrdered(rows) : "Scrub failed: " + rows;
@@@ -451,13 -389,10 +451,13 @@@
components.add(Component.STATS);
components.add(Component.SUMMARY);
components.add(Component.TOC);
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata);
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
- Scrubber scrubber = new Scrubber(cfs, sstable, false, true, true);
- scrubber.scrub();
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
+ {
+ scrubber.scrub();
+ }
cfs.loadNewSSTables();
assertEquals(7, countCells(cfs));
[10/10] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5c2a4ef2
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5c2a4ef2
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5c2a4ef2
Branch: refs/heads/trunk
Commit: 5c2a4ef280256d0d982d6322672893a8e2846988
Parents: 7863bd3 d436407
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Feb 11 08:29:22 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:29:22 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/compaction/CompactionManager.java | 12 +++---------
.../org/apache/cassandra/db/compaction/Scrubber.java | 12 ++++--------
.../org/apache/cassandra/tools/StandaloneScrubber.java | 2 +-
test/unit/org/apache/cassandra/db/ScrubTest.java | 12 ++++++------
5 files changed, 15 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5c2a4ef2/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5c2a4ef2/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5c2a4ef2/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5c2a4ef2/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
[07/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d7ff7f3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d7ff7f3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d7ff7f3
Branch: refs/heads/trunk
Commit: 7d7ff7f3cd317531a096e9f06c0bac9f5e139496
Parents: d719506 9654994
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Feb 11 08:25:14 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:25:14 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 12 +++---------
.../cassandra/db/compaction/Scrubber.java | 19 ++++++++++---------
.../cassandra/tools/StandaloneScrubber.java | 2 +-
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 63 -> 75 bytes
.../unit/org/apache/cassandra/db/ScrubTest.java | 14 +++++++-------
6 files changed, 22 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 318672f,9f51291..7565386
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,11 -1,9 +1,12 @@@
-2.1.14
+2.2.6
+ * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
+ * Fix paging on DISTINCT queries repeats result when first row in partition changes (CASSANDRA-10010)
+Merged from 2.1:
+ * Properly release sstable ref when doing offline scrub (CASSANDRA-10697)
* Improve nodetool status performance for large cluster (CASSANDRA-7238)
- * Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* Gossiper#isEnabled is not thread safe (CASSANDRA-11116)
* Avoid major compaction mixing repaired and unrepaired sstables in DTCS (CASSANDRA-11113)
+ * Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* test_bulk_round_trip_blogposts is failing occasionally (CASSANDRA-10938)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index c51ed7d,55b873a..8ca9852
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -327,15 -340,9 +327,9 @@@ public class CompactionManager implemen
}
}
- public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData) throws InterruptedException, ExecutionException
+ public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData)
+ throws InterruptedException, ExecutionException
{
- return performScrub(cfs, skipCorrupted, checkData, false);
- }
-
- public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData, final boolean offline)
- throws InterruptedException, ExecutionException
- {
- assert !cfs.isIndex();
return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
{
@Override
@@@ -345,30 -352,11 +339,30 @@@
}
@Override
- public void execute(SSTableReader input) throws IOException
+ public void execute(LifecycleTransaction input) throws IOException
{
- scrubOne(cfs, input, skipCorrupted, checkData, offline);
+ scrubOne(cfs, input, skipCorrupted, checkData);
}
- });
+ }, OperationType.SCRUB);
+ }
+
+ public AllSSTableOpStatus performVerify(final ColumnFamilyStore cfs, final boolean extendedVerify) throws InterruptedException, ExecutionException
+ {
+ assert !cfs.isIndex();
+ return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
+ {
+ @Override
+ public Iterable<SSTableReader> filterSSTables(LifecycleTransaction input)
+ {
+ return input.originals();
+ }
+
+ @Override
+ public void execute(LifecycleTransaction input) throws IOException
+ {
+ verifyOne(cfs, input.onlyOne(), extendedVerify);
+ }
+ }, OperationType.VERIFY);
}
public AllSSTableOpStatus performSSTableRewrite(final ColumnFamilyStore cfs, final boolean excludeCurrentVersion) throws InterruptedException, ExecutionException
@@@ -705,14 -683,14 +699,14 @@@
}
}
- private void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData, boolean offline) throws IOException
- private void scrubOne(ColumnFamilyStore cfs, SSTableReader sstable, boolean skipCorrupted, boolean checkData) throws IOException
++ private void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData) throws IOException
{
- Scrubber scrubber = new Scrubber(cfs, sstable, skipCorrupted, false, checkData);
+ CompactionInfo.Holder scrubInfo = null;
- try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, offline, checkData))
- CompactionInfo.Holder scrubInfo = scrubber.getScrubInfo();
- metrics.beginCompaction(scrubInfo);
- try
++ try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, checkData))
{
+ scrubInfo = scrubber.getScrubInfo();
+ metrics.beginCompaction(scrubInfo);
scrubber.scrub();
}
finally
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 9fd8560,8bfd37b..e9137e2
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -54,10 -49,9 +54,8 @@@ public class Scrubber implements Closea
private final RandomAccessReader dataFile;
private final RandomAccessReader indexFile;
private final ScrubInfo scrubInfo;
-
- private final boolean isOffline;
+ private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
- private final boolean isOffline;
-
private SSTableReader newSstable;
private SSTableReader newInOrderSstable;
@@@ -81,21 -75,19 +79,20 @@@
};
private final SortedSet<Row> outOfOrderRows = new TreeSet<>(rowComparator);
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException
- public Scrubber(ColumnFamilyStore cfs, SSTableReader sstable, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException
++ public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData) throws IOException
{
- this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
- this(cfs, sstable, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
++ this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), checkData);
}
- public Scrubber(ColumnFamilyStore cfs, SSTableReader sstable, boolean skipCorrupted, OutputHandler outputHandler, boolean isOffline, boolean checkData) throws IOException
+ @SuppressWarnings("resource")
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, OutputHandler outputHandler, boolean isOffline, boolean checkData) throws IOException
++ public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, OutputHandler outputHandler, boolean checkData) throws IOException
{
this.cfs = cfs;
- this.sstable = sstable;
+ this.transaction = transaction;
+ this.sstable = transaction.onlyOne();
this.outputHandler = outputHandler;
this.skipCorrupted = skipCorrupted;
-- this.isOffline = isOffline;
- this.validateColumns = checkData;
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata);
List<SSTableReader> toScrub = Collections.singletonList(sstable);
@@@ -105,7 -97,7 +102,7 @@@
throw new IOException("disk full");
// If we run scrub offline, we should never purge tombstone, as we cannot know if other sstable have data that the tombstone deletes.
-- this.controller = isOffline
++ this.controller = transaction.isOffline()
? new ScrubController(cfs)
: new CompactionController(cfs, Collections.singleton(sstable), CompactionManager.getDefaultGcBefore(cfs));
this.isCommutative = cfs.metadata.isCounter();
@@@ -126,7 -117,7 +123,7 @@@
// we'll also loop through the index at the same time, using the position from the index to recover if the
// row header (key or data size) is corrupt. (This means our position in the index file will be one row
// "ahead" of the data file.)
-- this.dataFile = isOffline
++ this.dataFile = transaction.isOffline()
? sstable.openDataReader()
: sstable.openDataReader(CompactionManager.instance.getRateLimiter());
@@@ -143,7 -134,9 +140,7 @@@
public void scrub()
{
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, isOffline))
- Set<SSTableReader> oldSSTable = Sets.newHashSet(sstable);
- SSTableRewriter writer = new SSTableRewriter(cfs, oldSSTable, sstable.maxDataAge, isOffline);
- try
++ try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline()))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
@@@ -299,13 -294,14 +296,15 @@@
{
// out of order rows, but no bad rows found - we can keep our repairedAt time
long repairedAt = badRows > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt;
- SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable);
- for (Row row : outOfOrderRows)
- inOrderWriter.append(row.key, row.cf);
- newInOrderSstable = inOrderWriter.closeAndOpenReader(sstable.maxDataAge);
- if (!isOffline)
- cfs.getDataTracker().addSSTables(Collections.singleton(newInOrderSstable));
- else if (newInOrderSstable != null)
+ try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable);)
+ {
+ for (Row row : outOfOrderRows)
+ inOrderWriter.append(row.key, row.cf);
+ newInOrderSstable = inOrderWriter.finish(-1, sstable.maxDataAge, true);
+ }
+ transaction.update(newInOrderSstable, false);
++ if (transaction.isOffline() && newInOrderSstable != null)
+ newInOrderSstable.selfRef().release();
outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrderRows.size(), sstable, newInOrderSstable));
}
@@@ -321,6 -320,8 +320,8 @@@
finally
{
controller.close();
- if (isOffline && newSstable != null)
++ if (transaction.isOffline() && newSstable != null)
+ newSstable.selfRef().release();
}
if (newSstable == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 3551b3d,fdf6c8d..a486a13
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@@ -119,10 -108,10 +119,10 @@@ public class StandaloneScrubbe
{
for (SSTableReader sstable : sstables)
{
- try
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable))
{
- Scrubber scrubber = new Scrubber(cfs, sstable, options.skipCorrupted, handler, true, !options.noValidate);
- try
+ txn.obsoleteOriginals(); // make sure originals are deleted and avoid NPE if index is missing, CASSANDRA-9591
- try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, true, !options.noValidate))
++ try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, !options.noValidate))
{
scrubber.scrub();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/ScrubTest.java
index b69a1f8,167671b..c0cde40
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@@ -166,21 -130,19 +166,21 @@@ public class ScrubTes
overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
// with skipCorrupted == false, the scrub is expected to fail
- Scrubber scrubber = new Scrubber(cfs, sstable, false, false, true);
- try
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, false, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
{
scrubber.scrub();
fail("Expected a CorruptSSTableException to be thrown");
}
catch (IOError err) {}
- // with skipCorrupted == true, the corrupt row will be skipped
+ // with skipCorrupted == true, the corrupt rows will be skipped
Scrubber.ScrubResult scrubResult;
- scrubber = new Scrubber(cfs, sstable, true, false, true);
- scrubResult = scrubber.scrubWithResult();
- scrubber.close();
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, false, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, true, true);)
+ {
+ scrubResult = scrubber.scrubWithResult();
+ }
assertNotNull(scrubResult);
@@@ -226,24 -188,20 +226,24 @@@
overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
// with skipCorrupted == false, the scrub is expected to fail
- Scrubber scrubber = new Scrubber(cfs, sstable, false, false, true);
- try
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, false, true))
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true))
{
+ // with skipCorrupted == true, the corrupt row will be skipped
scrubber.scrub();
fail("Expected a CorruptSSTableException to be thrown");
}
catch (IOError err) {}
- // with skipCorrupted == true, the corrupt row will be skipped
- scrubber = new Scrubber(cfs, sstable, true, false, true);
- scrubber.scrub();
- scrubber.close();
- assertEquals(1, cfs.getSSTables().size());
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, false, true))
++ Scrubber scrubber = new Scrubber(cfs, txn, true, true))
+ {
+ // with skipCorrupted == true, the corrupt row will be skipped
+ scrubber.scrub();
+ scrubber.close();
+ }
+ assertEquals(1, cfs.getSSTables().size());
// verify that we can read all of the rows, and there is now one less row
rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
assertEquals(1, rows.size());
@@@ -411,16 -353,11 +411,16 @@@
components.add(Component.STATS);
components.add(Component.SUMMARY);
components.add(Component.TOC);
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata);
- Scrubber scrubber = new Scrubber(cfs, sstable, false, true, true);
- scrubber.scrub();
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
+ if (sstable.last.compareTo(sstable.first) < 0)
+ sstable.last = sstable.first;
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
+ {
+ scrubber.scrub();
+ }
cfs.loadNewSSTables();
List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
assert isRowOrdered(rows) : "Scrub failed: " + rows;
@@@ -451,13 -389,10 +451,13 @@@
components.add(Component.STATS);
components.add(Component.SUMMARY);
components.add(Component.TOC);
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata);
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
- Scrubber scrubber = new Scrubber(cfs, sstable, false, true, true);
- scrubber.scrub();
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
+ {
+ scrubber.scrub();
+ }
cfs.loadNewSSTables();
assertEquals(7, countCells(cfs));
[09/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d4364075
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d4364075
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d4364075
Branch: refs/heads/trunk
Commit: d4364075dcc82c81990d98b39ed96d91ad40bf98
Parents: 89c558a 7d7ff7f
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Feb 11 08:28:26 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:28:26 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/compaction/CompactionManager.java | 12 +++---------
.../org/apache/cassandra/db/compaction/Scrubber.java | 12 ++++--------
.../org/apache/cassandra/tools/StandaloneScrubber.java | 2 +-
test/unit/org/apache/cassandra/db/ScrubTest.java | 12 ++++++------
5 files changed, 15 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 272c2f8,e9137e2..3dea9d9
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -57,8 -56,9 +57,6 @@@ public class Scrubber implements Closea
private final ScrubInfo scrubInfo;
private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
- private final boolean isOffline;
- private SSTableReader newSstable;
- private SSTableReader newInOrderSstable;
--
private int goodRows;
private int badRows;
private int emptyRows;
@@@ -70,37 -70,29 +68,35 @@@
private final OutputHandler outputHandler;
- private static final Comparator<Row> rowComparator = new Comparator<Row>()
+ private static final Comparator<Partition> partitionComparator = new Comparator<Partition>()
{
- public int compare(Row r1, Row r2)
+ public int compare(Partition r1, Partition r2)
{
- return r1.key.compareTo(r2.key);
+ return r1.partitionKey().compareTo(r2.partitionKey());
}
};
- private final SortedSet<Row> outOfOrderRows = new TreeSet<>(rowComparator);
+ private final SortedSet<Partition> outOfOrder = new TreeSet<>(partitionComparator);
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException
+ public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData) throws IOException
{
- this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
+ this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), checkData);
}
@SuppressWarnings("resource")
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, OutputHandler outputHandler, boolean checkData) throws IOException
+ public Scrubber(ColumnFamilyStore cfs,
+ LifecycleTransaction transaction,
+ boolean skipCorrupted,
+ OutputHandler outputHandler,
- boolean isOffline,
+ boolean checkData) throws IOException
{
this.cfs = cfs;
this.transaction = transaction;
this.sstable = transaction.onlyOne();
this.outputHandler = outputHandler;
this.skipCorrupted = skipCorrupted;
- this.isOffline = isOffline;
- this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata);
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata,
+ sstable.descriptor.version,
+ sstable.header);
List<SSTableReader> toScrub = Collections.singletonList(sstable);
@@@ -141,17 -137,10 +137,17 @@@
this.nextRowPositionFromIndex = 0;
}
+ private UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
+ {
+ return checkData ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
+ }
+
public void scrub()
{
+ List<SSTableReader> finished = new ArrayList<>();
+ boolean completed = false;
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = new SSTableRewriter(transaction, sstable.maxDataAge, isOffline))
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline()))
++ try (SSTableRewriter writer = new SSTableRewriter(transaction, sstable.maxDataAge, transaction.isOffline()))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/ScrubTest.java
index d5baec8,c0cde40..6dbbb1b
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@@ -145,7 -167,7 +145,7 @@@ public class ScrubTes
// with skipCorrupted == false, the scrub is expected to fail
try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, false, true))
- Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true))
{
scrubber.scrub();
fail("Expected a CorruptSSTableException to be thrown");
@@@ -155,7 -177,7 +155,7 @@@
// with skipCorrupted == true, the corrupt rows will be skipped
Scrubber.ScrubResult scrubResult;
try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, false, true))
- Scrubber scrubber = new Scrubber(cfs, txn, true, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, true, true))
{
scrubResult = scrubber.scrubWithResult();
}
@@@ -289,96 -340,146 +289,96 @@@
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
cfs.clearUnsafe();
- List<Row> rows;
-
// insert data and verify we get it back w/ range query
fillCF(cfs, 10);
- rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
- assertEquals(10, rows.size());
+ assertOrderedAll(cfs, 10);
- for (SSTableReader sstable : cfs.getSSTables())
+ for (SSTableReader sstable : cfs.getLiveSSTables())
new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)).delete();
- CompactionManager.instance.performScrub(cfs, false, true, true);
+ CompactionManager.instance.performScrub(cfs, false, true);
// check data is still there
- rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
- assertEquals(10, rows.size());
+ assertOrderedAll(cfs, 10);
}
@Test
public void testScrubOutOfOrder() throws Exception
{
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- String columnFamily = CF3;
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
- cfs.clearUnsafe();
-
- /*
- * Code used to generate an outOfOrder sstable. The test for out-of-order key in SSTableWriter must also be commented out.
- * The test also assumes an ordered partitioner.
- *
- ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.metadata);
- cf.addColumn(new BufferCell(ByteBufferUtil.bytes("someName"), ByteBufferUtil.bytes("someValue"), 0L));
-
- SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))),
- cfs.metadata.getIndexInterval(),
- cfs.metadata,
- cfs.partitioner,
- SSTableMetadata.createCollector(BytesType.instance));
- writer.append(Util.dk("a"), cf);
- writer.append(Util.dk("b"), cf);
- writer.append(Util.dk("z"), cf);
- writer.append(Util.dk("c"), cf);
- writer.append(Util.dk("y"), cf);
- writer.append(Util.dk("d"), cf);
- writer.finish();
- */
-
- String root = System.getProperty("corrupt-sstable-root");
- assert root != null;
-
- File rootDir = new File(root);
- assert rootDir.isDirectory();
- Descriptor desc = new Descriptor("jb", rootDir, KEYSPACE, columnFamily, 1, Descriptor.Type.FINAL, SSTableFormat.Type.LEGACY);
- CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname);
-
+ // This test assumes ByteOrderPartitioner to create out-of-order SSTable
+ IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner();
+ DatabaseDescriptor.setPartitionerUnsafe(new ByteOrderedPartitioner());
+
+ // Create out-of-order SSTable
+ File tempDir = File.createTempFile("ScrubTest.testScrubOutOfOrder", "").getParentFile();
+ // create ks/cf directory
+ File tempDataDir = new File(tempDir, String.join(File.separator, KEYSPACE, CF3));
+ tempDataDir.mkdirs();
try
{
- SSTableReader.open(desc, metadata);
- fail("SSTR validation should have caught the out-of-order rows");
- }
- catch (IllegalStateException ise) { /* this is expected */ }
-
- // open without validation for scrubbing
- Set<Component> components = new HashSet<>();
- components.add(Component.COMPRESSION_INFO);
- components.add(Component.DATA);
- components.add(Component.PRIMARY_INDEX);
- components.add(Component.FILTER);
- components.add(Component.STATS);
- components.add(Component.SUMMARY);
- components.add(Component.TOC);
-
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
- if (sstable.last.compareTo(sstable.first) < 0)
- sstable.last = sstable.first;
-
- try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
- {
- scrubber.scrub();
- }
- cfs.loadNewSSTables();
- List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
- assert isRowOrdered(rows) : "Scrub failed: " + rows;
- assert rows.size() == 6 : "Got " + rows.size();
- }
-
- @Test
- public void testScrub10791() throws Exception
- {
- // Table is created by StreamingTransferTest.testTransferRangeTombstones with CASSANDRA-10791 fix disabled.
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- String columnFamily = CFI1;
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
- cfs.clearUnsafe();
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ String columnFamily = CF3;
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
+ cfs.clearUnsafe();
+
+ List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
+ String filename = cfs.getSSTablePath(tempDataDir);
+ Descriptor desc = Descriptor.fromFilename(filename);
+
+ LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
+ try (SSTableTxnWriter writer = new SSTableTxnWriter(txn, createTestWriter(desc, (long) keys.size(), cfs.metadata, txn)))
+ {
- String root = System.getProperty("corrupt-sstable-root");
- assert root != null;
- File rootDir = new File(root);
- assert rootDir.isDirectory();
- Descriptor desc = new Descriptor("ka", rootDir, KEYSPACE, columnFamily, 2, Descriptor.Type.FINAL, SSTableFormat.Type.LEGACY);
-
- // open without validation for scrubbing
- Set<Component> components = new HashSet<>();
- components.add(Component.DATA);
- components.add(Component.PRIMARY_INDEX);
- components.add(Component.FILTER);
- components.add(Component.STATS);
- components.add(Component.SUMMARY);
- components.add(Component.TOC);
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
-
- try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
- {
- scrubber.scrub();
- }
+ for (String k : keys)
+ {
+ PartitionUpdate update = UpdateBuilder.create(cfs.metadata, Util.dk(k))
+ .newRow("someName").add("val", "someValue")
+ .build();
- cfs.loadNewSSTables();
- assertEquals(7, countCells(cfs));
- }
+ writer.append(update.unfilteredIterator());
+ }
+ writer.finish(false);
+ }
- private int countCells(ColumnFamilyStore cfs)
- {
- int cellCount = 0;
- for (SSTableReader sstable : cfs.getSSTables())
- {
- Iterator<OnDiskAtomIterator> it = sstable.getScanner();
- while (it.hasNext())
+ try
{
- Iterator<OnDiskAtom> itr = it.next();
- while (itr.hasNext())
- {
- ++cellCount;
- itr.next();
- }
+ SSTableReader.open(desc, cfs.metadata);
+ fail("SSTR validation should have caught the out-of-order rows");
+ }
+ catch (IllegalStateException ise)
+ { /* this is expected */ }
+
+ // open without validation for scrubbing
+ Set<Component> components = new HashSet<>();
+ if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
+ components.add(Component.COMPRESSION_INFO);
+ components.add(Component.DATA);
+ components.add(Component.PRIMARY_INDEX);
+ components.add(Component.FILTER);
+ components.add(Component.STATS);
+ components.add(Component.SUMMARY);
+ components.add(Component.TOC);
+
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
+ if (sstable.last.compareTo(sstable.first) < 0)
+ sstable.last = sstable.first;
+
+ try (LifecycleTransaction scrubTxn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, scrubTxn, false, true, true))
++ Scrubber scrubber = new Scrubber(cfs, scrubTxn, false, true))
+ {
+ scrubber.scrub();
}
+ LifecycleTransaction.waitForDeletions();
+ cfs.loadNewSSTables();
+ assertOrderedAll(cfs, 7);
+ }
+ finally
+ {
+ FileUtils.deleteRecursive(tempDataDir);
+ // reset partitioner
+ DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner);
}
- return cellCount;
}
private void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException
[08/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d4364075
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d4364075
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d4364075
Branch: refs/heads/cassandra-3.0
Commit: d4364075dcc82c81990d98b39ed96d91ad40bf98
Parents: 89c558a 7d7ff7f
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Feb 11 08:28:26 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:28:26 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/compaction/CompactionManager.java | 12 +++---------
.../org/apache/cassandra/db/compaction/Scrubber.java | 12 ++++--------
.../org/apache/cassandra/tools/StandaloneScrubber.java | 2 +-
test/unit/org/apache/cassandra/db/ScrubTest.java | 12 ++++++------
5 files changed, 15 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 272c2f8,e9137e2..3dea9d9
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -57,8 -56,9 +57,6 @@@ public class Scrubber implements Closea
private final ScrubInfo scrubInfo;
private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
- private final boolean isOffline;
- private SSTableReader newSstable;
- private SSTableReader newInOrderSstable;
--
private int goodRows;
private int badRows;
private int emptyRows;
@@@ -70,37 -70,29 +68,35 @@@
private final OutputHandler outputHandler;
- private static final Comparator<Row> rowComparator = new Comparator<Row>()
+ private static final Comparator<Partition> partitionComparator = new Comparator<Partition>()
{
- public int compare(Row r1, Row r2)
+ public int compare(Partition r1, Partition r2)
{
- return r1.key.compareTo(r2.key);
+ return r1.partitionKey().compareTo(r2.partitionKey());
}
};
- private final SortedSet<Row> outOfOrderRows = new TreeSet<>(rowComparator);
+ private final SortedSet<Partition> outOfOrder = new TreeSet<>(partitionComparator);
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException
+ public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData) throws IOException
{
- this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
+ this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), checkData);
}
@SuppressWarnings("resource")
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, OutputHandler outputHandler, boolean checkData) throws IOException
+ public Scrubber(ColumnFamilyStore cfs,
+ LifecycleTransaction transaction,
+ boolean skipCorrupted,
+ OutputHandler outputHandler,
- boolean isOffline,
+ boolean checkData) throws IOException
{
this.cfs = cfs;
this.transaction = transaction;
this.sstable = transaction.onlyOne();
this.outputHandler = outputHandler;
this.skipCorrupted = skipCorrupted;
- this.isOffline = isOffline;
- this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata);
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata,
+ sstable.descriptor.version,
+ sstable.header);
List<SSTableReader> toScrub = Collections.singletonList(sstable);
@@@ -141,17 -137,10 +137,17 @@@
this.nextRowPositionFromIndex = 0;
}
+ private UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename)
+ {
+ return checkData ? UnfilteredRowIterators.withValidation(iter, filename) : iter;
+ }
+
public void scrub()
{
+ List<SSTableReader> finished = new ArrayList<>();
+ boolean completed = false;
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = new SSTableRewriter(transaction, sstable.maxDataAge, isOffline))
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline()))
++ try (SSTableRewriter writer = new SSTableRewriter(transaction, sstable.maxDataAge, transaction.isOffline()))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4364075/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/ScrubTest.java
index d5baec8,c0cde40..6dbbb1b
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@@ -145,7 -167,7 +145,7 @@@ public class ScrubTes
// with skipCorrupted == false, the scrub is expected to fail
try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, false, true))
- Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true))
{
scrubber.scrub();
fail("Expected a CorruptSSTableException to be thrown");
@@@ -155,7 -177,7 +155,7 @@@
// with skipCorrupted == true, the corrupt rows will be skipped
Scrubber.ScrubResult scrubResult;
try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, false, true))
- Scrubber scrubber = new Scrubber(cfs, txn, true, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, true, true))
{
scrubResult = scrubber.scrubWithResult();
}
@@@ -289,96 -340,146 +289,96 @@@
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
cfs.clearUnsafe();
- List<Row> rows;
-
// insert data and verify we get it back w/ range query
fillCF(cfs, 10);
- rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
- assertEquals(10, rows.size());
+ assertOrderedAll(cfs, 10);
- for (SSTableReader sstable : cfs.getSSTables())
+ for (SSTableReader sstable : cfs.getLiveSSTables())
new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)).delete();
- CompactionManager.instance.performScrub(cfs, false, true, true);
+ CompactionManager.instance.performScrub(cfs, false, true);
// check data is still there
- rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
- assertEquals(10, rows.size());
+ assertOrderedAll(cfs, 10);
}
@Test
public void testScrubOutOfOrder() throws Exception
{
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- String columnFamily = CF3;
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
- cfs.clearUnsafe();
-
- /*
- * Code used to generate an outOfOrder sstable. The test for out-of-order key in SSTableWriter must also be commented out.
- * The test also assumes an ordered partitioner.
- *
- ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.metadata);
- cf.addColumn(new BufferCell(ByteBufferUtil.bytes("someName"), ByteBufferUtil.bytes("someValue"), 0L));
-
- SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))),
- cfs.metadata.getIndexInterval(),
- cfs.metadata,
- cfs.partitioner,
- SSTableMetadata.createCollector(BytesType.instance));
- writer.append(Util.dk("a"), cf);
- writer.append(Util.dk("b"), cf);
- writer.append(Util.dk("z"), cf);
- writer.append(Util.dk("c"), cf);
- writer.append(Util.dk("y"), cf);
- writer.append(Util.dk("d"), cf);
- writer.finish();
- */
-
- String root = System.getProperty("corrupt-sstable-root");
- assert root != null;
-
- File rootDir = new File(root);
- assert rootDir.isDirectory();
- Descriptor desc = new Descriptor("jb", rootDir, KEYSPACE, columnFamily, 1, Descriptor.Type.FINAL, SSTableFormat.Type.LEGACY);
- CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname);
-
+ // This test assumes ByteOrderPartitioner to create out-of-order SSTable
+ IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner();
+ DatabaseDescriptor.setPartitionerUnsafe(new ByteOrderedPartitioner());
+
+ // Create out-of-order SSTable
+ File tempDir = File.createTempFile("ScrubTest.testScrubOutOfOrder", "").getParentFile();
+ // create ks/cf directory
+ File tempDataDir = new File(tempDir, String.join(File.separator, KEYSPACE, CF3));
+ tempDataDir.mkdirs();
try
{
- SSTableReader.open(desc, metadata);
- fail("SSTR validation should have caught the out-of-order rows");
- }
- catch (IllegalStateException ise) { /* this is expected */ }
-
- // open without validation for scrubbing
- Set<Component> components = new HashSet<>();
- components.add(Component.COMPRESSION_INFO);
- components.add(Component.DATA);
- components.add(Component.PRIMARY_INDEX);
- components.add(Component.FILTER);
- components.add(Component.STATS);
- components.add(Component.SUMMARY);
- components.add(Component.TOC);
-
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
- if (sstable.last.compareTo(sstable.first) < 0)
- sstable.last = sstable.first;
-
- try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
- {
- scrubber.scrub();
- }
- cfs.loadNewSSTables();
- List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
- assert isRowOrdered(rows) : "Scrub failed: " + rows;
- assert rows.size() == 6 : "Got " + rows.size();
- }
-
- @Test
- public void testScrub10791() throws Exception
- {
- // Table is created by StreamingTransferTest.testTransferRangeTombstones with CASSANDRA-10791 fix disabled.
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- String columnFamily = CFI1;
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
- cfs.clearUnsafe();
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ String columnFamily = CF3;
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
+ cfs.clearUnsafe();
+
+ List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
+ String filename = cfs.getSSTablePath(tempDataDir);
+ Descriptor desc = Descriptor.fromFilename(filename);
+
+ LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
+ try (SSTableTxnWriter writer = new SSTableTxnWriter(txn, createTestWriter(desc, (long) keys.size(), cfs.metadata, txn)))
+ {
- String root = System.getProperty("corrupt-sstable-root");
- assert root != null;
- File rootDir = new File(root);
- assert rootDir.isDirectory();
- Descriptor desc = new Descriptor("ka", rootDir, KEYSPACE, columnFamily, 2, Descriptor.Type.FINAL, SSTableFormat.Type.LEGACY);
-
- // open without validation for scrubbing
- Set<Component> components = new HashSet<>();
- components.add(Component.DATA);
- components.add(Component.PRIMARY_INDEX);
- components.add(Component.FILTER);
- components.add(Component.STATS);
- components.add(Component.SUMMARY);
- components.add(Component.TOC);
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
-
- try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
- {
- scrubber.scrub();
- }
+ for (String k : keys)
+ {
+ PartitionUpdate update = UpdateBuilder.create(cfs.metadata, Util.dk(k))
+ .newRow("someName").add("val", "someValue")
+ .build();
- cfs.loadNewSSTables();
- assertEquals(7, countCells(cfs));
- }
+ writer.append(update.unfilteredIterator());
+ }
+ writer.finish(false);
+ }
- private int countCells(ColumnFamilyStore cfs)
- {
- int cellCount = 0;
- for (SSTableReader sstable : cfs.getSSTables())
- {
- Iterator<OnDiskAtomIterator> it = sstable.getScanner();
- while (it.hasNext())
+ try
{
- Iterator<OnDiskAtom> itr = it.next();
- while (itr.hasNext())
- {
- ++cellCount;
- itr.next();
- }
+ SSTableReader.open(desc, cfs.metadata);
+ fail("SSTR validation should have caught the out-of-order rows");
+ }
+ catch (IllegalStateException ise)
+ { /* this is expected */ }
+
+ // open without validation for scrubbing
+ Set<Component> components = new HashSet<>();
+ if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
+ components.add(Component.COMPRESSION_INFO);
+ components.add(Component.DATA);
+ components.add(Component.PRIMARY_INDEX);
+ components.add(Component.FILTER);
+ components.add(Component.STATS);
+ components.add(Component.SUMMARY);
+ components.add(Component.TOC);
+
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
+ if (sstable.last.compareTo(sstable.first) < 0)
+ sstable.last = sstable.first;
+
+ try (LifecycleTransaction scrubTxn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, scrubTxn, false, true, true))
++ Scrubber scrubber = new Scrubber(cfs, scrubTxn, false, true))
+ {
+ scrubber.scrub();
}
+ LifecycleTransaction.waitForDeletions();
+ cfs.loadNewSSTables();
+ assertOrderedAll(cfs, 7);
+ }
+ finally
+ {
+ FileUtils.deleteRecursive(tempDataDir);
+ // reset partitioner
+ DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner);
}
- return cellCount;
}
private void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException
[02/10] cassandra git commit: Properly release sstable reference when
doing offline scrub
Posted by ma...@apache.org.
Properly release sstable reference when doing offline scrub
Patch by marcuse; reviewed by Stefania for CASSANDRA-10697
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/96549946
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/96549946
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/96549946
Branch: refs/heads/cassandra-2.2
Commit: 96549946fe994ea8311e72e5bf6f51c2124f7bb4
Parents: 6c6b7e4
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Feb 9 09:53:59 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:20:47 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/compaction/Scrubber.java | 4 ++++
.../apache/cassandra/tools/StandaloneScrubber.java | 5 ++++-
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 63 -> 75 bytes
.../Keyspace1-StandardInteger1-ka-2-Summary.db | Bin 80 -> 112 bytes
5 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3bbef11..9f51291 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.14
+ * Properly release sstable ref when doing offline scrub (CASSANDRA-10697)
* Improve nodetool status performance for large cluster (CASSANDRA-7238)
* Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* Gossiper#isEnabled is not thread safe (CASSANDRA-11116)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index e02f901..8bfd37b 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -300,6 +300,8 @@ public class Scrubber implements Closeable
newInOrderSstable = inOrderWriter.closeAndOpenReader(sstable.maxDataAge);
if (!isOffline)
cfs.getDataTracker().addSSTables(Collections.singleton(newInOrderSstable));
+ else if (newInOrderSstable != null)
+ newInOrderSstable.selfRef().release();
outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrderRows.size(), sstable, newInOrderSstable));
}
@@ -318,6 +320,8 @@ public class Scrubber implements Closeable
finally
{
controller.close();
+ if (isOffline && newSstable != null)
+ newSstable.selfRef().release();
}
if (newSstable == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 79ba0bd..fdf6c8d 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -122,13 +122,16 @@ public class StandaloneScrubber
// Remove the sstable (it's been copied by scrub and snapshotted)
sstable.markObsolete(null);
- sstable.selfRef().release();
}
catch (Exception e)
{
System.err.println(String.format("Error scrubbing %s: %s", sstable, e.getMessage()));
e.printStackTrace(System.err);
}
+ finally
+ {
+ sstable.selfRef().release();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
index 7621f07..6eb7650 100644
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db b/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db
index 22cfa6a..190922a 100644
Binary files a/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db and b/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db differ
[05/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d7ff7f3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d7ff7f3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d7ff7f3
Branch: refs/heads/cassandra-3.0
Commit: 7d7ff7f3cd317531a096e9f06c0bac9f5e139496
Parents: d719506 9654994
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Feb 11 08:25:14 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:25:14 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 12 +++---------
.../cassandra/db/compaction/Scrubber.java | 19 ++++++++++---------
.../cassandra/tools/StandaloneScrubber.java | 2 +-
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 63 -> 75 bytes
.../unit/org/apache/cassandra/db/ScrubTest.java | 14 +++++++-------
6 files changed, 22 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 318672f,9f51291..7565386
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,11 -1,9 +1,12 @@@
-2.1.14
+2.2.6
+ * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
+ * Fix paging on DISTINCT queries repeats result when first row in partition changes (CASSANDRA-10010)
+Merged from 2.1:
+ * Properly release sstable ref when doing offline scrub (CASSANDRA-10697)
* Improve nodetool status performance for large cluster (CASSANDRA-7238)
- * Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* Gossiper#isEnabled is not thread safe (CASSANDRA-11116)
* Avoid major compaction mixing repaired and unrepaired sstables in DTCS (CASSANDRA-11113)
+ * Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* test_bulk_round_trip_blogposts is failing occasionally (CASSANDRA-10938)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index c51ed7d,55b873a..8ca9852
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -327,15 -340,9 +327,9 @@@ public class CompactionManager implemen
}
}
- public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData) throws InterruptedException, ExecutionException
+ public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData)
+ throws InterruptedException, ExecutionException
{
- return performScrub(cfs, skipCorrupted, checkData, false);
- }
-
- public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData, final boolean offline)
- throws InterruptedException, ExecutionException
- {
- assert !cfs.isIndex();
return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
{
@Override
@@@ -345,30 -352,11 +339,30 @@@
}
@Override
- public void execute(SSTableReader input) throws IOException
+ public void execute(LifecycleTransaction input) throws IOException
{
- scrubOne(cfs, input, skipCorrupted, checkData, offline);
+ scrubOne(cfs, input, skipCorrupted, checkData);
}
- });
+ }, OperationType.SCRUB);
+ }
+
+ public AllSSTableOpStatus performVerify(final ColumnFamilyStore cfs, final boolean extendedVerify) throws InterruptedException, ExecutionException
+ {
+ assert !cfs.isIndex();
+ return parallelAllSSTableOperation(cfs, new OneSSTableOperation()
+ {
+ @Override
+ public Iterable<SSTableReader> filterSSTables(LifecycleTransaction input)
+ {
+ return input.originals();
+ }
+
+ @Override
+ public void execute(LifecycleTransaction input) throws IOException
+ {
+ verifyOne(cfs, input.onlyOne(), extendedVerify);
+ }
+ }, OperationType.VERIFY);
}
public AllSSTableOpStatus performSSTableRewrite(final ColumnFamilyStore cfs, final boolean excludeCurrentVersion) throws InterruptedException, ExecutionException
@@@ -705,14 -683,14 +699,14 @@@
}
}
- private void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData, boolean offline) throws IOException
- private void scrubOne(ColumnFamilyStore cfs, SSTableReader sstable, boolean skipCorrupted, boolean checkData) throws IOException
++ private void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData) throws IOException
{
- Scrubber scrubber = new Scrubber(cfs, sstable, skipCorrupted, false, checkData);
+ CompactionInfo.Holder scrubInfo = null;
- try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, offline, checkData))
- CompactionInfo.Holder scrubInfo = scrubber.getScrubInfo();
- metrics.beginCompaction(scrubInfo);
- try
++ try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, checkData))
{
+ scrubInfo = scrubber.getScrubInfo();
+ metrics.beginCompaction(scrubInfo);
scrubber.scrub();
}
finally
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java
index 9fd8560,8bfd37b..e9137e2
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@@ -54,10 -49,9 +54,8 @@@ public class Scrubber implements Closea
private final RandomAccessReader dataFile;
private final RandomAccessReader indexFile;
private final ScrubInfo scrubInfo;
-
- private final boolean isOffline;
+ private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
- private final boolean isOffline;
-
private SSTableReader newSstable;
private SSTableReader newInOrderSstable;
@@@ -81,21 -75,19 +79,20 @@@
};
private final SortedSet<Row> outOfOrderRows = new TreeSet<>(rowComparator);
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException
- public Scrubber(ColumnFamilyStore cfs, SSTableReader sstable, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException
++ public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData) throws IOException
{
- this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
- this(cfs, sstable, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
++ this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), checkData);
}
- public Scrubber(ColumnFamilyStore cfs, SSTableReader sstable, boolean skipCorrupted, OutputHandler outputHandler, boolean isOffline, boolean checkData) throws IOException
+ @SuppressWarnings("resource")
- public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, OutputHandler outputHandler, boolean isOffline, boolean checkData) throws IOException
++ public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, OutputHandler outputHandler, boolean checkData) throws IOException
{
this.cfs = cfs;
- this.sstable = sstable;
+ this.transaction = transaction;
+ this.sstable = transaction.onlyOne();
this.outputHandler = outputHandler;
this.skipCorrupted = skipCorrupted;
-- this.isOffline = isOffline;
- this.validateColumns = checkData;
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata);
List<SSTableReader> toScrub = Collections.singletonList(sstable);
@@@ -105,7 -97,7 +102,7 @@@
throw new IOException("disk full");
// If we run scrub offline, we should never purge tombstone, as we cannot know if other sstable have data that the tombstone deletes.
-- this.controller = isOffline
++ this.controller = transaction.isOffline()
? new ScrubController(cfs)
: new CompactionController(cfs, Collections.singleton(sstable), CompactionManager.getDefaultGcBefore(cfs));
this.isCommutative = cfs.metadata.isCounter();
@@@ -126,7 -117,7 +123,7 @@@
// we'll also loop through the index at the same time, using the position from the index to recover if the
// row header (key or data size) is corrupt. (This means our position in the index file will be one row
// "ahead" of the data file.)
-- this.dataFile = isOffline
++ this.dataFile = transaction.isOffline()
? sstable.openDataReader()
: sstable.openDataReader(CompactionManager.instance.getRateLimiter());
@@@ -143,7 -134,9 +140,7 @@@
public void scrub()
{
outputHandler.output(String.format("Scrubbing %s (%s bytes)", sstable, dataFile.length()));
- try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, isOffline))
- Set<SSTableReader> oldSSTable = Sets.newHashSet(sstable);
- SSTableRewriter writer = new SSTableRewriter(cfs, oldSSTable, sstable.maxDataAge, isOffline);
- try
++ try (SSTableRewriter writer = new SSTableRewriter(cfs, transaction, sstable.maxDataAge, transaction.isOffline()))
{
nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
if (indexAvailable())
@@@ -299,13 -294,14 +296,15 @@@
{
// out of order rows, but no bad rows found - we can keep our repairedAt time
long repairedAt = badRows > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt;
- SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable);
- for (Row row : outOfOrderRows)
- inOrderWriter.append(row.key, row.cf);
- newInOrderSstable = inOrderWriter.closeAndOpenReader(sstable.maxDataAge);
- if (!isOffline)
- cfs.getDataTracker().addSSTables(Collections.singleton(newInOrderSstable));
- else if (newInOrderSstable != null)
+ try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable);)
+ {
+ for (Row row : outOfOrderRows)
+ inOrderWriter.append(row.key, row.cf);
+ newInOrderSstable = inOrderWriter.finish(-1, sstable.maxDataAge, true);
+ }
+ transaction.update(newInOrderSstable, false);
++ if (transaction.isOffline() && newInOrderSstable != null)
+ newInOrderSstable.selfRef().release();
outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrderRows.size(), sstable, newInOrderSstable));
}
@@@ -321,6 -320,8 +320,8 @@@
finally
{
controller.close();
- if (isOffline && newSstable != null)
++ if (transaction.isOffline() && newSstable != null)
+ newSstable.selfRef().release();
}
if (newSstable == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 3551b3d,fdf6c8d..a486a13
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@@ -119,10 -108,10 +119,10 @@@ public class StandaloneScrubbe
{
for (SSTableReader sstable : sstables)
{
- try
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable))
{
- Scrubber scrubber = new Scrubber(cfs, sstable, options.skipCorrupted, handler, true, !options.noValidate);
- try
+ txn.obsoleteOriginals(); // make sure originals are deleted and avoid NPE if index is missing, CASSANDRA-9591
- try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, true, !options.noValidate))
++ try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, !options.noValidate))
{
scrubber.scrub();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d7ff7f3/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/ScrubTest.java
index b69a1f8,167671b..c0cde40
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@@ -166,21 -130,19 +166,21 @@@ public class ScrubTes
overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
// with skipCorrupted == false, the scrub is expected to fail
- Scrubber scrubber = new Scrubber(cfs, sstable, false, false, true);
- try
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, false, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
{
scrubber.scrub();
fail("Expected a CorruptSSTableException to be thrown");
}
catch (IOError err) {}
- // with skipCorrupted == true, the corrupt row will be skipped
+ // with skipCorrupted == true, the corrupt rows will be skipped
Scrubber.ScrubResult scrubResult;
- scrubber = new Scrubber(cfs, sstable, true, false, true);
- scrubResult = scrubber.scrubWithResult();
- scrubber.close();
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, false, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, true, true);)
+ {
+ scrubResult = scrubber.scrubWithResult();
+ }
assertNotNull(scrubResult);
@@@ -226,24 -188,20 +226,24 @@@
overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"));
// with skipCorrupted == false, the scrub is expected to fail
- Scrubber scrubber = new Scrubber(cfs, sstable, false, false, true);
- try
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, false, false, true))
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true))
{
+ // with skipCorrupted == true, the corrupt row will be skipped
scrubber.scrub();
fail("Expected a CorruptSSTableException to be thrown");
}
catch (IOError err) {}
- // with skipCorrupted == true, the corrupt row will be skipped
- scrubber = new Scrubber(cfs, sstable, true, false, true);
- scrubber.scrub();
- scrubber.close();
- assertEquals(1, cfs.getSSTables().size());
+ try (LifecycleTransaction txn = cfs.getTracker().tryModify(Arrays.asList(sstable), OperationType.SCRUB);
- Scrubber scrubber = new Scrubber(cfs, txn, true, false, true))
++ Scrubber scrubber = new Scrubber(cfs, txn, true, true))
+ {
+ // with skipCorrupted == true, the corrupt row will be skipped
+ scrubber.scrub();
+ scrubber.close();
+ }
+ assertEquals(1, cfs.getSSTables().size());
// verify that we can read all of the rows, and there is now one less row
rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
assertEquals(1, rows.size());
@@@ -411,16 -353,11 +411,16 @@@
components.add(Component.STATS);
components.add(Component.SUMMARY);
components.add(Component.TOC);
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata);
- Scrubber scrubber = new Scrubber(cfs, sstable, false, true, true);
- scrubber.scrub();
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
+ if (sstable.last.compareTo(sstable.first) < 0)
+ sstable.last = sstable.first;
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
+ {
+ scrubber.scrub();
+ }
cfs.loadNewSSTables();
List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
assert isRowOrdered(rows) : "Scrub failed: " + rows;
@@@ -451,13 -389,10 +451,13 @@@
components.add(Component.STATS);
components.add(Component.SUMMARY);
components.add(Component.TOC);
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata);
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
- Scrubber scrubber = new Scrubber(cfs, sstable, false, true, true);
- scrubber.scrub();
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, true, true);)
++ Scrubber scrubber = new Scrubber(cfs, txn, false, true);)
+ {
+ scrubber.scrub();
+ }
cfs.loadNewSSTables();
assertEquals(7, countCells(cfs));
[03/10] cassandra git commit: Properly release sstable reference when
doing offline scrub
Posted by ma...@apache.org.
Properly release sstable reference when doing offline scrub
Patch by marcuse; reviewed by Stefania for CASSANDRA-10697
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/96549946
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/96549946
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/96549946
Branch: refs/heads/cassandra-3.0
Commit: 96549946fe994ea8311e72e5bf6f51c2124f7bb4
Parents: 6c6b7e4
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Feb 9 09:53:59 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Feb 11 08:20:47 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/compaction/Scrubber.java | 4 ++++
.../apache/cassandra/tools/StandaloneScrubber.java | 5 ++++-
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 63 -> 75 bytes
.../Keyspace1-StandardInteger1-ka-2-Summary.db | Bin 80 -> 112 bytes
5 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3bbef11..9f51291 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.14
+ * Properly release sstable ref when doing offline scrub (CASSANDRA-10697)
* Improve nodetool status performance for large cluster (CASSANDRA-7238)
* Make it clear what DTCS timestamp_resolution is used for (CASSANDRA-11041)
* Gossiper#isEnabled is not thread safe (CASSANDRA-11116)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index e02f901..8bfd37b 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -300,6 +300,8 @@ public class Scrubber implements Closeable
newInOrderSstable = inOrderWriter.closeAndOpenReader(sstable.maxDataAge);
if (!isOffline)
cfs.getDataTracker().addSSTables(Collections.singleton(newInOrderSstable));
+ else if (newInOrderSstable != null)
+ newInOrderSstable.selfRef().release();
outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrderRows.size(), sstable, newInOrderSstable));
}
@@ -318,6 +320,8 @@ public class Scrubber implements Closeable
finally
{
controller.close();
+ if (isOffline && newSstable != null)
+ newSstable.selfRef().release();
}
if (newSstable == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 79ba0bd..fdf6c8d 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -122,13 +122,16 @@ public class StandaloneScrubber
// Remove the sstable (it's been copied by scrub and snapshotted)
sstable.markObsolete(null);
- sstable.selfRef().release();
}
catch (Exception e)
{
System.err.println(String.format("Error scrubbing %s: %s", sstable, e.getMessage()));
e.printStackTrace(System.err);
}
+ finally
+ {
+ sstable.selfRef().release();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
index 7621f07..6eb7650 100644
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/96549946/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db b/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db
index 22cfa6a..190922a 100644
Binary files a/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db and b/test/data/corrupt-sstables/Keyspace1-StandardInteger1-ka-2-Summary.db differ