You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2013/09/26 20:21:59 UTC
[1/3] git commit: add back ScrubTest patch by Tyler Hobbs;
reviewed by jbellis for CASSANDRA-5730
Updated Branches:
refs/heads/cassandra-2.0 b5c23cf74 -> d49303078
refs/heads/trunk 5832cc839 -> 2c7b61b76
add back ScrubTest
patch by Tyler Hobbs; reviewed by jbellis for CASSANDRA-5730
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d4930307
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d4930307
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d4930307
Branch: refs/heads/cassandra-2.0
Commit: d49303078459b0b2e0d40d9f79660ccdc0dc1fe0
Parents: b5c23cf
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Sep 26 13:18:02 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Sep 26 13:19:04 2013 -0500
----------------------------------------------------------------------
.../Keyspace1-Standard3-ja-1-CRC.db | Bin 8 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Data.db | Bin 354 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Digest.sha1 | 1 -
.../Keyspace1-Standard3-ja-1-Filter.db | Bin 176 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Index.db | Bin 90 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Statistics.db | Bin 4377 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Summary.db | Bin 83 -> 0 bytes
.../Keyspace1-Standard3-ja-1-TOC.txt | 8 -
.../Keyspace1-Standard3-jb-1-CompressionInfo.db | Bin 0 -> 43 bytes
.../Keyspace1-Standard3-jb-1-Data.db | Bin 0 -> 133 bytes
.../Keyspace1-Standard3-jb-1-Filter.db | Bin 0 -> 24 bytes
.../Keyspace1-Standard3-jb-1-Index.db | Bin 0 -> 90 bytes
.../Keyspace1-Standard3-jb-1-Statistics.db | Bin 0 -> 4390 bytes
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 0 -> 71 bytes
.../Keyspace1-Standard3-jb-1-TOC.txt | 7 +
.../unit/org/apache/cassandra/db/ScrubTest.java | 206 +++++++++++++++++++
16 files changed, 213 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db
deleted file mode 100644
index 3cc23b0..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db
deleted file mode 100644
index 70e64e0..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1 b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1
deleted file mode 100644
index c53d478..0000000
--- a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a9fbab0c12f097cfbf91a7b8731a20363daef547 Keyspace1-Standard3-ja-1-Data.db
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db
deleted file mode 100644
index df0734d..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db
deleted file mode 100644
index 5da2914..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db
deleted file mode 100644
index 28e250b..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db
deleted file mode 100644
index 105e42c..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt
deleted file mode 100644
index 39df2ce..0000000
--- a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Statistics.db
-Digest.sha1
-Filter.db
-Data.db
-Index.db
-Summary.db
-CRC.db
-TOC.txt
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db
new file mode 100644
index 0000000..44d2e59
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db
new file mode 100644
index 0000000..f75c4e6
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db
new file mode 100644
index 0000000..8f0a999
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db
new file mode 100644
index 0000000..da84fbc
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db
new file mode 100644
index 0000000..0762615
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
new file mode 100644
index 0000000..376ca9d
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt
new file mode 100644
index 0000000..cf6efa8
--- /dev/null
+++ b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt
@@ -0,0 +1,7 @@
+CompressionInfo.db
+TOC.txt
+Filter.db
+Statistics.db
+Data.db
+Summary.db
+Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
new file mode 100644
index 0000000..f09aab3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -0,0 +1,206 @@
+package org.apache.cassandra.db;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.compaction.Scrubber;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.Util.column;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class ScrubTest extends SchemaLoader
+{
+ public String KEYSPACE = "Keyspace1";
+ public String CF = "Standard1";
+ public String CF3 = "Standard2";
+
+ @Test
+ public void testScrubOneRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ List<Row> rows;
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 1);
+ rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assertEquals(1, rows.size());
+
+ CompactionManager.instance.performScrub(cfs);
+
+ // check data is still there
+ rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assertEquals(1, rows.size());
+ }
+
+ @Test
+ public void testScrubDeletedRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF3);
+
+ ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(KEYSPACE, CF3);
+ cf.delete(new DeletionInfo(0, 1)); // expired tombstone
+ RowMutation rm = new RowMutation(KEYSPACE, ByteBufferUtil.bytes(1), cf);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ CompactionManager.instance.performScrub(cfs);
+ assert cfs.getSSTables().isEmpty();
+ }
+
+ @Test
+ public void testScrubMultiRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ List<Row> rows;
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 10);
+ rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assertEquals(10, rows.size());
+
+ CompactionManager.instance.performScrub(cfs);
+
+ // check data is still there
+ rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assertEquals(10, rows.size());
+ }
+
+ @Test
+ public void testScrubOutOfOrder() throws Exception
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ String columnFamily = "Standard3";
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
+
+ /*
+ * Code used to generate an outOfOrder sstable. The test for out-of-order key in SSTableWriter must also be commented out.
+ * The test also assumes an ordered partitioner.
+ *
+ ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.metadata);
+ cf.addColumn(new Column(ByteBufferUtil.bytes("someName"), ByteBufferUtil.bytes("someValue"), 0L));
+
+ SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))),
+ cfs.metadata.getIndexInterval(),
+ cfs.metadata,
+ cfs.partitioner,
+ SSTableMetadata.createCollector(BytesType.instance));
+ writer.append(Util.dk("a"), cf);
+ writer.append(Util.dk("b"), cf);
+ writer.append(Util.dk("z"), cf);
+ writer.append(Util.dk("c"), cf);
+ writer.append(Util.dk("y"), cf);
+ writer.append(Util.dk("d"), cf);
+ writer.closeAndOpenReader();
+ */
+
+
+ String root = System.getProperty("corrupt-sstable-root");
+ assert root != null;
+ File rootDir = new File(root);
+ assert rootDir.isDirectory();
+ Descriptor desc = new Descriptor(new Descriptor.Version("jb"), rootDir, KEYSPACE, columnFamily, 1, false);
+ CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname);
+
+ try
+ {
+ SSTableReader.open(desc, metadata);
+ fail("SSTR validation should have caught the out-of-order rows");
+ }
+ catch (IllegalStateException ise) { /* this is expected */ }
+
+ // open without validation for scrubbing
+ Set<Component> components = new HashSet<>();
+ components.add(Component.COMPRESSION_INFO);
+ components.add(Component.DATA);
+ components.add(Component.PRIMARY_INDEX);
+ components.add(Component.FILTER);
+ components.add(Component.STATS);
+ components.add(Component.SUMMARY);
+ components.add(Component.TOC);
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata);
+
+ Scrubber scrubber = new Scrubber(cfs, sstable);
+ scrubber.scrub();
+
+ cfs.loadNewSSTables();
+ List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assert isRowOrdered(rows) : "Scrub failed: " + rows;
+ assert rows.size() == 6 : "Got " + rows.size();
+ }
+
+ private static boolean isRowOrdered(List<Row> rows)
+ {
+ DecoratedKey prev = null;
+ for (Row row : rows)
+ {
+ if (prev != null && prev.compareTo(row.key) > 0)
+ return false;
+ prev = row.key;
+ }
+ return true;
+ }
+
+ protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) throws ExecutionException, InterruptedException, IOException
+ {
+ for (int i = 0; i < rowsPerSSTable; i++)
+ {
+ String key = String.valueOf(i);
+ // create a row and update the birthdate value, test that the index query fetches the new version
+ ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(KEYSPACE, CF);
+ cf.addColumn(column("c1", "1", 1L));
+ cf.addColumn(column("c2", "2", 1L));
+ RowMutation rm = new RowMutation(KEYSPACE, ByteBufferUtil.bytes(key), cf);
+ rm.applyUnsafe();
+ }
+
+ cfs.forceBlockingFlush();
+ }
+}
[3/3] git commit: Merge branch 'cassandra-2.0' into trunk
Posted by jb...@apache.org.
Merge branch 'cassandra-2.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2c7b61b7
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2c7b61b7
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2c7b61b7
Branch: refs/heads/trunk
Commit: 2c7b61b76ec034afe4267fdaecd0905db16b40eb
Parents: 5832cc8 d493030
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Sep 26 13:20:25 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Sep 26 13:20:25 2013 -0500
----------------------------------------------------------------------
.../Keyspace1-Standard3-ja-1-CRC.db | Bin 8 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Data.db | Bin 354 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Digest.sha1 | 1 -
.../Keyspace1-Standard3-ja-1-Filter.db | Bin 176 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Index.db | Bin 90 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Statistics.db | Bin 4377 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Summary.db | Bin 83 -> 0 bytes
.../Keyspace1-Standard3-ja-1-TOC.txt | 8 -
.../Keyspace1-Standard3-jb-1-CompressionInfo.db | Bin 0 -> 43 bytes
.../Keyspace1-Standard3-jb-1-Data.db | Bin 0 -> 133 bytes
.../Keyspace1-Standard3-jb-1-Filter.db | Bin 0 -> 24 bytes
.../Keyspace1-Standard3-jb-1-Index.db | Bin 0 -> 90 bytes
.../Keyspace1-Standard3-jb-1-Statistics.db | Bin 0 -> 4390 bytes
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 0 -> 71 bytes
.../Keyspace1-Standard3-jb-1-TOC.txt | 7 +
.../unit/org/apache/cassandra/db/ScrubTest.java | 206 +++++++++++++++++++
16 files changed, 213 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[2/3] git commit: add back ScrubTest patch by Tyler Hobbs;
reviewed by jbellis for CASSANDRA-5730
Posted by jb...@apache.org.
add back ScrubTest
patch by Tyler Hobbs; reviewed by jbellis for CASSANDRA-5730
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d4930307
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d4930307
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d4930307
Branch: refs/heads/trunk
Commit: d49303078459b0b2e0d40d9f79660ccdc0dc1fe0
Parents: b5c23cf
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Sep 26 13:18:02 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Sep 26 13:19:04 2013 -0500
----------------------------------------------------------------------
.../Keyspace1-Standard3-ja-1-CRC.db | Bin 8 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Data.db | Bin 354 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Digest.sha1 | 1 -
.../Keyspace1-Standard3-ja-1-Filter.db | Bin 176 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Index.db | Bin 90 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Statistics.db | Bin 4377 -> 0 bytes
.../Keyspace1-Standard3-ja-1-Summary.db | Bin 83 -> 0 bytes
.../Keyspace1-Standard3-ja-1-TOC.txt | 8 -
.../Keyspace1-Standard3-jb-1-CompressionInfo.db | Bin 0 -> 43 bytes
.../Keyspace1-Standard3-jb-1-Data.db | Bin 0 -> 133 bytes
.../Keyspace1-Standard3-jb-1-Filter.db | Bin 0 -> 24 bytes
.../Keyspace1-Standard3-jb-1-Index.db | Bin 0 -> 90 bytes
.../Keyspace1-Standard3-jb-1-Statistics.db | Bin 0 -> 4390 bytes
.../Keyspace1-Standard3-jb-1-Summary.db | Bin 0 -> 71 bytes
.../Keyspace1-Standard3-jb-1-TOC.txt | 7 +
.../unit/org/apache/cassandra/db/ScrubTest.java | 206 +++++++++++++++++++
16 files changed, 213 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db
deleted file mode 100644
index 3cc23b0..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db
deleted file mode 100644
index 70e64e0..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1 b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1
deleted file mode 100644
index c53d478..0000000
--- a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a9fbab0c12f097cfbf91a7b8731a20363daef547 Keyspace1-Standard3-ja-1-Data.db
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db
deleted file mode 100644
index df0734d..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db
deleted file mode 100644
index 5da2914..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db
deleted file mode 100644
index 28e250b..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db
deleted file mode 100644
index 105e42c..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt
deleted file mode 100644
index 39df2ce..0000000
--- a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Statistics.db
-Digest.sha1
-Filter.db
-Data.db
-Index.db
-Summary.db
-CRC.db
-TOC.txt
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db
new file mode 100644
index 0000000..44d2e59
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db
new file mode 100644
index 0000000..f75c4e6
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db
new file mode 100644
index 0000000..8f0a999
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db
new file mode 100644
index 0000000..da84fbc
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db
new file mode 100644
index 0000000..0762615
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
new file mode 100644
index 0000000..376ca9d
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt
new file mode 100644
index 0000000..cf6efa8
--- /dev/null
+++ b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt
@@ -0,0 +1,7 @@
+CompressionInfo.db
+TOC.txt
+Filter.db
+Statistics.db
+Data.db
+Summary.db
+Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
new file mode 100644
index 0000000..f09aab3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -0,0 +1,206 @@
+package org.apache.cassandra.db;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.compaction.Scrubber;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.Util.column;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class ScrubTest extends SchemaLoader
+{
+ public String KEYSPACE = "Keyspace1";
+ public String CF = "Standard1";
+ public String CF3 = "Standard2";
+
+ @Test
+ public void testScrubOneRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ List<Row> rows;
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 1);
+ rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assertEquals(1, rows.size());
+
+ CompactionManager.instance.performScrub(cfs);
+
+ // check data is still there
+ rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assertEquals(1, rows.size());
+ }
+
+ @Test
+ public void testScrubDeletedRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF3);
+
+ ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(KEYSPACE, CF3);
+ cf.delete(new DeletionInfo(0, 1)); // expired tombstone
+ RowMutation rm = new RowMutation(KEYSPACE, ByteBufferUtil.bytes(1), cf);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ CompactionManager.instance.performScrub(cfs);
+ assert cfs.getSSTables().isEmpty();
+ }
+
+ @Test
+ public void testScrubMultiRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+ List<Row> rows;
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 10);
+ rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assertEquals(10, rows.size());
+
+ CompactionManager.instance.performScrub(cfs);
+
+ // check data is still there
+ rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assertEquals(10, rows.size());
+ }
+
+ @Test
+ public void testScrubOutOfOrder() throws Exception
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ String columnFamily = "Standard3";
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
+
+ /*
+ * Code used to generate an outOfOrder sstable. The test for out-of-order key in SSTableWriter must also be commented out.
+ * The test also assumes an ordered partitioner.
+ *
+ ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.metadata);
+ cf.addColumn(new Column(ByteBufferUtil.bytes("someName"), ByteBufferUtil.bytes("someValue"), 0L));
+
+ SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))),
+ cfs.metadata.getIndexInterval(),
+ cfs.metadata,
+ cfs.partitioner,
+ SSTableMetadata.createCollector(BytesType.instance));
+ writer.append(Util.dk("a"), cf);
+ writer.append(Util.dk("b"), cf);
+ writer.append(Util.dk("z"), cf);
+ writer.append(Util.dk("c"), cf);
+ writer.append(Util.dk("y"), cf);
+ writer.append(Util.dk("d"), cf);
+ writer.closeAndOpenReader();
+ */
+
+
+ String root = System.getProperty("corrupt-sstable-root");
+ assert root != null;
+ File rootDir = new File(root);
+ assert rootDir.isDirectory();
+ Descriptor desc = new Descriptor(new Descriptor.Version("jb"), rootDir, KEYSPACE, columnFamily, 1, false);
+ CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname);
+
+ try
+ {
+ SSTableReader.open(desc, metadata);
+ fail("SSTR validation should have caught the out-of-order rows");
+ }
+ catch (IllegalStateException ise) { /* this is expected */ }
+
+ // open without validation for scrubbing
+ Set<Component> components = new HashSet<>();
+ components.add(Component.COMPRESSION_INFO);
+ components.add(Component.DATA);
+ components.add(Component.PRIMARY_INDEX);
+ components.add(Component.FILTER);
+ components.add(Component.STATS);
+ components.add(Component.SUMMARY);
+ components.add(Component.TOC);
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata);
+
+ Scrubber scrubber = new Scrubber(cfs, sstable);
+ scrubber.scrub();
+
+ cfs.loadNewSSTables();
+ List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+ assert isRowOrdered(rows) : "Scrub failed: " + rows;
+ assert rows.size() == 6 : "Got " + rows.size();
+ }
+
+ private static boolean isRowOrdered(List<Row> rows)
+ {
+ DecoratedKey prev = null;
+ for (Row row : rows)
+ {
+ if (prev != null && prev.compareTo(row.key) > 0)
+ return false;
+ prev = row.key;
+ }
+ return true;
+ }
+
+ protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) throws ExecutionException, InterruptedException, IOException
+ {
+ for (int i = 0; i < rowsPerSSTable; i++)
+ {
+ String key = String.valueOf(i);
+ // create a row and update the birthdate value, test that the index query fetches the new version
+ ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(KEYSPACE, CF);
+ cf.addColumn(column("c1", "1", 1L));
+ cf.addColumn(column("c2", "2", 1L));
+ RowMutation rm = new RowMutation(KEYSPACE, ByteBufferUtil.bytes(key), cf);
+ rm.applyUnsafe();
+ }
+
+ cfs.forceBlockingFlush();
+ }
+}