You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yu...@apache.org on 2015/07/30 17:12:41 UTC
cassandra git commit: ScrubTest should generate out-of-order SSTable
on the fly
Repository: cassandra
Updated Branches:
refs/heads/trunk b090ed693 -> df3b6027b
ScrubTest should generate out-of-order SSTable on the fly
with some other ScrubTest fixes.
patch by yukim; reviewed by Stefania Alborghetti for CASSANDRA-9880
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/df3b6027
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/df3b6027
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/df3b6027
Branch: refs/heads/trunk
Commit: df3b6027bc90c658963e7bf72933b1507c6c6beb
Parents: b090ed6
Author: Yuki Morishita <yu...@apache.org>
Authored: Thu Jul 30 10:07:25 2015 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Thu Jul 30 10:09:07 2015 -0500
----------------------------------------------------------------------
build.xml | 4 -
.../cassandra/db/compaction/Scrubber.java | 7 +-
.../io/sstable/format/SSTableReader.java | 1 -
.../cassandra/tools/StandaloneScrubber.java | 2 +-
test/data/corrupt-sstables/la-1-big-CRC.db | Bin 8 -> 0 bytes
test/data/corrupt-sstables/la-1-big-Data.db | Bin 259 -> 0 bytes
.../corrupt-sstables/la-1-big-Digest.adler32 | 1 -
test/data/corrupt-sstables/la-1-big-Filter.db | Bin 24 -> 0 bytes
test/data/corrupt-sstables/la-1-big-Index.db | Bin 105 -> 0 bytes
.../corrupt-sstables/la-1-big-Statistics.db | Bin 4645 -> 0 bytes
test/data/corrupt-sstables/la-1-big-Summary.db | Bin 83 -> 0 bytes
test/data/corrupt-sstables/la-1-big-TOC.txt | 8 -
.../unit/org/apache/cassandra/db/ScrubTest.java | 180 ++++++++++---------
13 files changed, 97 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index b56e379..bc433ae 100644
--- a/build.xml
+++ b/build.xml
@@ -1281,7 +1281,6 @@
<testmacro inputdir="${test.unit.src}" exclude="**/pig/*.java" timeout="${test.timeout}">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
- <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
<jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
@@ -1302,7 +1301,6 @@
<testmacrohelper inputdir="${test.unit.src}" filelist="@{test.file.list}" poffset="@{testlist.offset}" exclude="**/*.java" timeout="${test.timeout}">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
- <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
<jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
@@ -1323,7 +1321,6 @@
exclude="**/*.java" timeout="${test.timeout}" testtag="compression">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
- <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
<jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
<jvmarg value="-Dcassandra.test.compression=true"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
@@ -1355,7 +1352,6 @@
<test name="${test.name}" methods="${test.methods}"/>
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
- <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
<jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index c853157..81e307a 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -59,7 +59,6 @@ public class Scrubber implements Closeable
private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
private final boolean isOffline;
- private final boolean keepOriginals;
private SSTableReader newSstable;
private SSTableReader newInOrderSstable;
@@ -86,7 +85,7 @@ public class Scrubber implements Closeable
public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException
{
- this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData, false);
+ this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
}
@SuppressWarnings("resource")
@@ -95,8 +94,7 @@ public class Scrubber implements Closeable
boolean skipCorrupted,
OutputHandler outputHandler,
boolean isOffline,
- boolean checkData,
- boolean keepOriginals) throws IOException
+ boolean checkData) throws IOException
{
this.cfs = cfs;
this.transaction = transaction;
@@ -104,7 +102,6 @@ public class Scrubber implements Closeable
this.outputHandler = outputHandler;
this.skipCorrupted = skipCorrupted;
this.isOffline = isOffline;
- this.keepOriginals = keepOriginals;
this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata,
sstable.descriptor.version,
sstable.header);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index e9ac200..6d39d2d 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -1251,7 +1251,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
{
if (this.first.compareTo(this.last) > 0)
{
- selfRef().release();
throw new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index c00d036..9388d98 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -121,7 +121,7 @@ public class StandaloneScrubber
try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable))
{
txn.obsoleteOriginals(); // make sure originals are deleted and avoid NPE if index is missing, CASSANDRA-9591
- try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, true, !options.noValidate, false))
+ try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, true, !options.noValidate))
{
scrubber.scrub();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-CRC.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-CRC.db b/test/data/corrupt-sstables/la-1-big-CRC.db
deleted file mode 100644
index 1a0c525..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-CRC.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Data.db b/test/data/corrupt-sstables/la-1-big-Data.db
deleted file mode 100644
index e6c5eb9..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Data.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Digest.adler32
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Digest.adler32 b/test/data/corrupt-sstables/la-1-big-Digest.adler32
deleted file mode 100644
index 93deb45..0000000
--- a/test/data/corrupt-sstables/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-3942663153
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Filter.db b/test/data/corrupt-sstables/la-1-big-Filter.db
deleted file mode 100644
index 709d2ea..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Filter.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Index.db b/test/data/corrupt-sstables/la-1-big-Index.db
deleted file mode 100644
index 6e5e352..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Index.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Statistics.db b/test/data/corrupt-sstables/la-1-big-Statistics.db
deleted file mode 100644
index 15220e0..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Statistics.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Summary.db b/test/data/corrupt-sstables/la-1-big-Summary.db
deleted file mode 100644
index 732f27c..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Summary.db and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-TOC.txt b/test/data/corrupt-sstables/la-1-big-TOC.txt
deleted file mode 100644
index 9ad71ef..0000000
--- a/test/data/corrupt-sstables/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Statistics.db
-CRC.db
-TOC.txt
-Data.db
-Index.db
-Summary.db
-Digest.adler32
-Filter.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
index 9b62590..ee51a4d 100644
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -18,54 +18,52 @@
*/
package org.apache.cassandra.db;
+import java.io.File;
+import java.io.IOError;
+import java.io.IOException;
+import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ExecutionException;
-import java.io.File;
-import java.io.IOError;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.UpdateBuilder;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.RequestExecutionException;
-import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.commons.lang3.StringUtils;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.cassandra.config.*;
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.compaction.Scrubber;
import org.apache.cassandra.db.index.SecondaryIndex;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableRewriter;
-import org.apache.cassandra.OrderedJUnit4ClassRunner;
-import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.io.sstable.SSTableTxnWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.OutputHandler;
import static org.junit.Assert.*;
import static org.junit.Assume.assumeTrue;
@@ -306,74 +304,84 @@ public class ScrubTest
@Test
public void testScrubOutOfOrder() throws Exception
{
- CompactionManager.instance.disableAutoCompaction();
- Keyspace keyspace = Keyspace.open(KEYSPACE);
- String columnFamily = CF3;
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
- cfs.clearUnsafe();
-
- /*
- * Code used to generate an outOfOrder sstable. The test for out-of-order key in BigTableWriter must also be commented out.
- * The test also assumes an ordered partitioner.
- List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
- SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))),
- SSTableWriter writer = SSTableWriter.create(cfs.metadata,
- Descriptor.fromFilename(filename),
- keys.size(),
- 0L,
- 0,
- DatabaseDescriptor.getPartitioner(),
- SerializationHeader.make(cfs.metadata, Collections.emptyList()));
-
- for (String k : keys)
+ // This test assumes ByteOrderPartitioner to create out-of-order SSTable
+ IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner();
+ DatabaseDescriptor.setPartitioner(new ByteOrderedPartitioner());
+
+ // Create out-of-order SSTable
+ File tempDir = File.createTempFile("ScrubTest.testScrubOutOfOrder", "").getParentFile();
+ // create ks/cf directory
+ File tempDataDir = new File(tempDir, String.join(File.separator, KEYSPACE, CF3));
+ tempDataDir.mkdirs();
+ try
{
- RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0L, Util.dk(k));
- PartitionUpdate update = builder.clustering("someName")
- .add("val", "someValue")
- .buildUpdate();
-
- writer.append(update.unfilteredIterator());
- }
- writer.finish(false);
- */
+ CompactionManager.instance.disableAutoCompaction();
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ String columnFamily = CF3;
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
+ cfs.clearUnsafe();
+
+ List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
+ String filename = cfs.getSSTablePath(tempDataDir);
+ Descriptor desc = Descriptor.fromFilename(filename);
+
+ try (SSTableTxnWriter writer = SSTableTxnWriter.create(desc,
+ keys.size(),
+ 0L,
+ 0,
+ SerializationHeader.make(cfs.metadata,
+ Collections.emptyList())))
+ {
- String root = System.getProperty("corrupt-sstable-root");
- assert root != null;
+ for (String k : keys)
+ {
+ PartitionUpdate update = UpdateBuilder.create(cfs.metadata, Util.dk(k))
+ .newRow("someName").add("val", "someValue")
+ .build();
- File rootDir = new File(root);
- assert rootDir.isDirectory();
- Descriptor desc = new Descriptor("la", rootDir, KEYSPACE, columnFamily, 1, SSTableFormat.Type.BIG);
- CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname);
+ writer.append(update.unfilteredIterator());
+ }
+ writer.finish(false);
+ }
- try
- {
- SSTableReader.open(desc, metadata);
- fail("SSTR validation should have caught the out-of-order rows");
+ try
+ {
+ SSTableReader.open(desc, cfs.metadata);
+ fail("SSTR validation should have caught the out-of-order rows");
+ }
+ catch (IllegalStateException ise)
+ { /* this is expected */ }
+
+ // open without validation for scrubbing
+ Set<Component> components = new HashSet<>();
+ if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
+ components.add(Component.COMPRESSION_INFO);
+ components.add(Component.DATA);
+ components.add(Component.PRIMARY_INDEX);
+ components.add(Component.FILTER);
+ components.add(Component.STATS);
+ components.add(Component.SUMMARY);
+ components.add(Component.TOC);
+
+ SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
+ if (sstable.last.compareTo(sstable.first) < 0)
+ sstable.last = sstable.first;
+
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
+ Scrubber scrubber = new Scrubber(cfs, txn, false, true, true))
+ {
+ scrubber.scrub();
+ }
+ cfs.loadNewSSTables();
+ assertOrderedAll(cfs, 7);
+ sstable.selfRef().release();
}
- catch (IllegalStateException ise) { /* this is expected */ }
-
- // open without validation for scrubbing
- Set<Component> components = new HashSet<>();
- //components.add(Component.COMPRESSION_INFO);
- components.add(Component.DATA);
- components.add(Component.PRIMARY_INDEX);
- components.add(Component.FILTER);
- components.add(Component.STATS);
- components.add(Component.SUMMARY);
- components.add(Component.TOC);
-
- SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
- if (sstable.last.compareTo(sstable.first) < 0)
- sstable.last = sstable.first;
-
- try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable);
- Scrubber scrubber = new Scrubber(cfs, txn, false, new OutputHandler.LogOutput(), true, true, true))
+ finally
{
- scrubber.scrub();
+ FileUtils.deleteRecursive(tempDataDir);
+ // reset partitioner
+ DatabaseDescriptor.setPartitioner(oldPartitioner);
}
- cfs.loadNewSSTables();
- assertOrderedAll(cfs, 7);
- sstable.selfRef().release();
}
private void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException