You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2013/04/24 03:15:22 UTC
[10/15] git commit: resurrect ScrubTest
resurrect ScrubTest
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3d557240
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3d557240
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3d557240
Branch: refs/heads/trunk
Commit: 3d557240ae85f626852fe7eb14bf8cfb1077d1b2
Parents: 302267e
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Apr 19 13:43:26 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Tue Apr 23 10:24:18 2013 -0500
----------------------------------------------------------------------
.../corrupt-sstables/Keyspace1-Super5-f-2-Data.db | Bin 530200 -> 0 bytes
.../Keyspace1-Super5-f-2-Filter.db | Bin 496 -> 0 bytes
.../corrupt-sstables/Keyspace1-Super5-f-2-Index.db | Bin 1300 -> 0 bytes
.../Keyspace1-Super5-f-2-Statistics.db | Bin 4264 -> 0 bytes
test/unit/org/apache/cassandra/db/ScrubTest.java | 211 +++++++++++++++
5 files changed, 211 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Data.db b/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Data.db
deleted file mode 100644
index 0f95b07..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Data.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Filter.db b/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Filter.db
deleted file mode 100644
index 88aac99..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Filter.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Index.db b/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Index.db
deleted file mode 100644
index a7787c5..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Index.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Statistics.db
deleted file mode 100644
index 6312c71..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Statistics.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
new file mode 100644
index 0000000..26f0e78
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -0,0 +1,211 @@
+package org.apache.cassandra.db;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CLibrary;
+
+import static org.apache.cassandra.Util.column;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class ScrubTest extends SchemaLoader
+{
+ public String TABLE = "Keyspace1";
+ public String CF = "Standard1";
+ public String CF2 = "Super5";
+ public String CF3 = "Standard2";
+
+ public String copySSTables(String cf) throws IOException
+ {
+ String root = System.getProperty("corrupt-sstable-root");
+ assert root != null;
+ File rootDir = new File(root);
+ assert rootDir.isDirectory();
+
+ File destDir = Directories.create(TABLE, cf).getDirectoryForNewSSTables(1);
+
+ String corruptSSTableName = null;
+
+ FileUtils.createDirectory(destDir);
+ for (File srcFile : rootDir.listFiles())
+ {
+ if (srcFile.getName().equals(".svn"))
+ continue;
+ if (!srcFile.getName().contains(cf))
+ continue;
+ File destFile = new File(destDir, srcFile.getName());
+ CLibrary.createHardLink(srcFile, destFile);
+
+ assert destFile.exists() : destFile.getAbsoluteFile();
+
+ if(destFile.getName().endsWith("Data.db"))
+ corruptSSTableName = destFile.getCanonicalPath();
+ }
+
+ assert corruptSSTableName != null;
+ return corruptSSTableName;
+ }
+
+ @Test
+ public void testScrubOneRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Table table = Table.open(TABLE);
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(CF);
+
+ List<Row> rows;
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 1);
+ rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(), null);
+ assertEquals(1, rows.size());
+
+ CompactionManager.instance.performScrub(cfs);
+
+ // check data is still there
+ rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(), null);
+ assertEquals(1, rows.size());
+ }
+
+ @Test
+ public void testScrubDeletedRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Table table = Table.open(TABLE);
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(CF3);
+
+ RowMutation rm;
+ rm = new RowMutation(TABLE, ByteBufferUtil.bytes(1));
+ ColumnFamily cf = ColumnFamily.create(TABLE, CF3);
+ cf.delete(new DeletionInfo(0, 1)); // expired tombstone
+ rm.add(cf);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ CompactionManager.instance.performScrub(cfs);
+ assert cfs.getSSTables().isEmpty();
+ }
+
+ @Test
+ public void testScrubMultiRow() throws IOException, ExecutionException, InterruptedException, ConfigurationException
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Table table = Table.open(TABLE);
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(CF);
+
+ List<Row> rows;
+
+ // insert data and verify we get it back w/ range query
+ fillCF(cfs, 10);
+ rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(), null);
+ assertEquals(10, rows.size());
+
+ CompactionManager.instance.performScrub(cfs);
+
+ // check data is still there
+ rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(), null);
+ assertEquals(10, rows.size());
+ }
+
+ @Test
+ public void testScubOutOfOrder() throws Exception
+ {
+ CompactionManager.instance.disableAutoCompaction();
+ Table table = Table.open(TABLE);
+ String columnFamily = "Standard3";
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(columnFamily);
+
+ /*
+ * Code used to generate an outOfOrder sstable. The test for out-of-order key in SSTableWriter must also be commented out.
+ * The test also assumes an ordered partitioner.
+ *
+ * ColumnFamily cf = ColumnFamily.create(TABLE, columnFamily);
+ * cf.addColumn(new Column(ByteBufferUtil.bytes("someName"), ByteBufferUtil.bytes("someValue"), 0L));
+
+ * SSTableWriter writer = cfs.createCompactionWriter((long)DatabaseDescriptor.getIndexInterval(), new File("."), Collections.<SSTableReader>emptyList());
+ * writer.append(Util.dk("a"), cf);
+ * writer.append(Util.dk("b"), cf);
+ * writer.append(Util.dk("z"), cf);
+ * writer.append(Util.dk("c"), cf);
+ * writer.append(Util.dk("y"), cf);
+ * writer.append(Util.dk("d"), cf);
+ * writer.closeAndOpenReader();
+ */
+
+ copySSTables(columnFamily);
+ cfs.loadNewSSTables();
+ assert cfs.getSSTables().size() > 0;
+
+ List<Row> rows;
+ rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(), null);
+ assert !isRowOrdered(rows) : "'corrupt' test file actually was not";
+
+ CompactionManager.instance.performScrub(cfs);
+ rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(), null);
+ assert isRowOrdered(rows) : "Scrub failed: " + rows;
+ assert rows.size() == 6: "Got " + rows.size();
+ }
+
+ private static boolean isRowOrdered(List<Row> rows)
+ {
+ DecoratedKey prev = null;
+ for (Row row : rows)
+ {
+ if (prev != null && prev.compareTo(row.key) > 0)
+ return false;
+ prev = row.key;
+ }
+ return true;
+ }
+
+ protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) throws ExecutionException, InterruptedException, IOException
+ {
+ for (int i = 0; i < rowsPerSSTable; i++)
+ {
+ String key = String.valueOf(i);
+ // create a row and update the birthdate value, test that the index query fetches the new version
+ RowMutation rm;
+ rm = new RowMutation(TABLE, ByteBufferUtil.bytes(key));
+ ColumnFamily cf = ColumnFamily.create(TABLE, CF);
+ cf.addColumn(column("c1", "1", 1L));
+ cf.addColumn(column("c2", "2", 1L));
+ rm.add(cf);
+ rm.applyUnsafe();
+ }
+
+ cfs.forceBlockingFlush();
+ }
+}