You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by gd...@apache.org on 2010/12/29 15:16:56 UTC
svn commit: r1053638 - in /cassandra/branches/cassandra-0.7/test:
data/legacy-sstables/b/Keyspace1/ data/legacy-sstables/e/
data/legacy-sstables/e/Keyspace1/ long/org/apache/cassandra/db/
unit/org/apache/cassandra/io/sstable/ unit/org/apache/cassandra/...
Author: gdusbabek
Date: Wed Dec 29 14:16:55 2010
New Revision: 1053638
URL: http://svn.apache.org/viewvc?rev=1053638&view=rev
Log:
row level coverage in LegacySSTableTest. patch by stuhood, reviewed by gdusbabek. CASSANDRA-1822
Added:
cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/
cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/
cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Data.db (with props)
cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Filter.db (with props)
cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Index.db (with props)
cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Statistics.db (with props)
Modified:
cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db
cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db
cassandra/branches/cassandra-0.7/test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
Modified: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db?rev=1053638&r1=1053637&r2=1053638&view=diff
==============================================================================
Files cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db (original) and cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Data.db Wed Dec 29 14:16:55 2010 differ
Modified: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db?rev=1053638&r1=1053637&r2=1053638&view=diff
==============================================================================
Files cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db (original) and cassandra/branches/cassandra-0.7/test/data/legacy-sstables/b/Keyspace1/Standard1-b-0-Index.db Wed Dec 29 14:16:55 2010 differ
Added: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Data.db
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Data.db?rev=1053638&view=auto
==============================================================================
Binary file - no diff available.
Propchange: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Data.db
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Filter.db
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Filter.db?rev=1053638&view=auto
==============================================================================
Binary file - no diff available.
Propchange: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Filter.db
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Index.db
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Index.db?rev=1053638&view=auto
==============================================================================
Binary file - no diff available.
Propchange: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Index.db
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Statistics.db
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Statistics.db?rev=1053638&view=auto
==============================================================================
Binary file - no diff available.
Propchange: cassandra/branches/cassandra-0.7/test/data/legacy-sstables/e/Keyspace1/Standard1-e-0-Statistics.db
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: cassandra/branches/cassandra-0.7/test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java?rev=1053638&r1=1053637&r2=1053638&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java (original)
+++ cassandra/branches/cassandra-0.7/test/long/org/apache/cassandra/db/LongCompactionSpeedTest.java Wed Dec 29 14:16:55 2010
@@ -86,7 +86,7 @@ public class LongCompactionSpeedTest ext
}
rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols));
}
- SSTableReader sstable = SSTableUtils.writeSSTable(rows);
+ SSTableReader sstable = SSTableUtils.prepare().write(rows);
sstables.add(sstable);
store.addSSTable(sstable);
}
Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java?rev=1053638&r1=1053637&r2=1053638&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java Wed Dec 29 14:16:55 2010
@@ -22,15 +22,11 @@ package org.apache.cassandra.io.sstable;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import org.apache.cassandra.CleanupHelper;
-import org.apache.cassandra.io.util.BufferedRandomAccessFile;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.columniterator.SSTableNamesIterator;
import org.apache.cassandra.utils.FBUtilities;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -45,7 +41,7 @@ public class LegacySSTableTest extends C
public static final String KSNAME = "Keyspace1";
public static final String CFNAME = "Standard1";
- public static Map<ByteBuffer, ByteBuffer> TEST_DATA;
+ public static Set<String> TEST_DATA;
public static File LEGACY_SSTABLE_ROOT;
@BeforeClass
@@ -56,11 +52,9 @@ public class LegacySSTableTest extends C
LEGACY_SSTABLE_ROOT = new File(scp).getAbsoluteFile();
assert LEGACY_SSTABLE_ROOT.isDirectory();
- TEST_DATA = new HashMap<ByteBuffer,ByteBuffer>();
+ TEST_DATA = new HashSet<String>();
for (int i = 100; i < 1000; ++i)
- {
- TEST_DATA.put(ByteBuffer.wrap(Integer.toString(i).getBytes()), ByteBuffer.wrap(("Avinash Lakshman is a good man: " + i).getBytes()));
- }
+ TEST_DATA.add(Integer.toString(i));
}
/**
@@ -83,44 +77,39 @@ public class LegacySSTableTest extends C
Descriptor dest = getDescriptor(Descriptor.CURRENT_VERSION);
assert dest.directory.mkdirs() : "Could not create " + dest.directory + ". Might it already exist?";
- SSTableReader ssTable = SSTableUtils.writeRawSSTable(new File(dest.filenameFor(SSTable.COMPONENT_DATA)),
- KSNAME,
- CFNAME,
- TEST_DATA);
- assert ssTable.desc.generation == 0 :
+ SSTableReader ssTable = SSTableUtils.prepare().ks(KSNAME).cf(CFNAME).dest(dest).write(TEST_DATA);
+ assert ssTable.descriptor.generation == 0 :
"In order to create a generation 0 sstable, please run this test alone.";
System.out.println(">>> Wrote " + dest);
}
*/
@Test
- public void testVersions() throws IOException
+ public void testVersions() throws Throwable
{
for (File version : LEGACY_SSTABLE_ROOT.listFiles())
if (Descriptor.versionValidate(version.getName()))
testVersion(version.getName());
}
- public void testVersion(String version)
+ public void testVersion(String version) throws Throwable
{
try
{
SSTableReader reader = SSTableReader.open(getDescriptor(version));
-
- List<ByteBuffer> keys = new ArrayList<ByteBuffer>(TEST_DATA.keySet());
- Collections.shuffle(keys);
- BufferedRandomAccessFile file = new BufferedRandomAccessFile(reader.getFilename(), "r");
- for (ByteBuffer key : keys)
+ for (String keystring : TEST_DATA)
{
- // confirm that the bloom filter does not reject any keys
- file.seek(reader.getPosition(reader.partitioner.decorateKey(key), SSTableReader.Operator.EQ));
- assert key.equals( FBUtilities.readShortByteArray(file));
+ ByteBuffer key = ByteBuffer.wrap(keystring.getBytes());
+ // confirm that the bloom filter does not reject any keys/names
+ DecoratedKey dk = reader.partitioner.decorateKey(key);
+ SSTableNamesIterator iter = new SSTableNamesIterator(reader, dk, FBUtilities.singleton(key));
+ assert iter.next().name().equals(key);
}
}
catch (Throwable e)
{
System.err.println("Failed to read " + version);
- e.printStackTrace(System.err);
+ throw e;
}
}
}
Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java?rev=1053638&r1=1053637&r2=1053638&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java Wed Dec 29 14:16:55 2010
@@ -40,7 +40,7 @@ public class SSTableTest extends Cleanup
Map<ByteBuffer, ByteBuffer> map = new HashMap<ByteBuffer,ByteBuffer>();
map.put(key, bytes);
- SSTableReader ssTable = SSTableUtils.writeRawSSTable("Keyspace1", "Standard1", map);
+ SSTableReader ssTable = SSTableUtils.prepare().cf("Standard1").writeRaw(map);
// verify
verifySingle(ssTable, bytes, key);
@@ -68,7 +68,7 @@ public class SSTableTest extends Cleanup
}
// write
- SSTableReader ssTable = SSTableUtils.writeRawSSTable("Keyspace1", "Standard2", map);
+ SSTableReader ssTable = SSTableUtils.prepare().cf("Standard2").writeRaw(map);
// verify
verifyMany(ssTable, map);
Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java?rev=1053638&r1=1053637&r2=1053638&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java Wed Dec 29 14:16:55 2010
@@ -22,12 +22,7 @@ package org.apache.cassandra.io.sstable;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import java.util.*;
import org.apache.cassandra.db.Column;
import org.apache.cassandra.db.ColumnFamily;
@@ -52,6 +47,11 @@ public class SSTableUtils
public static File tempSSTableFile(String tablename, String cfname) throws IOException
{
+ return tempSSTableFile(tablename, cfname, 0);
+ }
+
+ public static File tempSSTableFile(String tablename, String cfname, int generation) throws IOException
+ {
File tempdir = File.createTempFile(tablename, cfname);
if(!tempdir.delete() || !tempdir.mkdir())
throw new IOException("Temporary directory creation failed.");
@@ -59,49 +59,105 @@ public class SSTableUtils
File tabledir = new File(tempdir, tablename);
tabledir.mkdir();
tabledir.deleteOnExit();
- File datafile = new File(new Descriptor(tabledir, tablename, cfname, 0, false).filenameFor("Data.db"));
+ File datafile = new File(new Descriptor(tabledir, tablename, cfname, generation, false).filenameFor("Data.db"));
if (!datafile.createNewFile())
throw new IOException("unable to create file " + datafile);
datafile.deleteOnExit();
return datafile;
}
- public static SSTableReader writeSSTable(Set<String> keys) throws IOException
+ /**
+ * @return A Context with chainable methods to configure and write a SSTable.
+ */
+ public static Context prepare()
{
- Map<String, ColumnFamily> map = new HashMap<String, ColumnFamily>();
- for (String key : keys)
- {
- ColumnFamily cf = ColumnFamily.create(TABLENAME, CFNAME);
- cf.addColumn(new Column(ByteBuffer.wrap(key.getBytes()), ByteBuffer.wrap(key.getBytes()), 0));
- map.put(key, cf);
- }
- return writeSSTable(map);
+ return new Context();
}
- public static SSTableReader writeSSTable(Map<String, ColumnFamily> entries) throws IOException
+ public static class Context
{
- Map<ByteBuffer, ByteBuffer> map = new HashMap<ByteBuffer, ByteBuffer>();
- for (Map.Entry<String, ColumnFamily> entry : entries.entrySet())
+ private String ksname = TABLENAME;
+ private String cfname = CFNAME;
+ private Descriptor dest = null;
+ private boolean cleanup = true;
+ private int generation = 0;
+
+ Context() {}
+
+ public Context ks(String ksname)
{
- DataOutputBuffer buffer = new DataOutputBuffer();
- ColumnFamily.serializer().serializeWithIndexes(entry.getValue(), buffer);
- map.put(ByteBuffer.wrap(entry.getKey().getBytes()), ByteBuffer.wrap(Arrays.copyOf(buffer.getData(), buffer.getLength())));
+ this.ksname = ksname;
+ return this;
}
- return writeRawSSTable(TABLENAME, CFNAME, map);
- }
- public static SSTableReader writeRawSSTable(String tablename, String cfname, Map<ByteBuffer, ByteBuffer> entries) throws IOException
- {
- File datafile = tempSSTableFile(tablename, cfname);
- SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size());
- SortedMap<DecoratedKey, ByteBuffer> sortedEntries = new TreeMap<DecoratedKey, ByteBuffer>();
- for (Map.Entry<ByteBuffer, ByteBuffer> entry : entries.entrySet())
- sortedEntries.put(writer.partitioner.decorateKey(entry.getKey()), entry.getValue());
- for (Map.Entry<DecoratedKey, ByteBuffer> entry : sortedEntries.entrySet())
- writer.append(entry.getKey(), entry.getValue());
- new File(writer.descriptor.filenameFor(Component.PRIMARY_INDEX)).deleteOnExit();
- new File(writer.descriptor.filenameFor(Component.FILTER)).deleteOnExit();
- return writer.closeAndOpenReader();
- }
+ public Context cf(String cfname)
+ {
+ this.cfname = cfname;
+ return this;
+ }
+ /**
+ * Set an alternate path for the written SSTable: if unset, the SSTable will
+ * be cleaned up on JVM exit.
+ */
+ public Context dest(Descriptor dest)
+ {
+ this.dest = dest;
+ this.cleanup = false;
+ return this;
+ }
+
+ /**
+ * Sets the generation number for the generated SSTable. Ignored if "dest()" is set.
+ */
+ public Context generation(int generation)
+ {
+ this.generation = generation;
+ return this;
+ }
+
+ public SSTableReader write(Set<String> keys) throws IOException
+ {
+ Map<String, ColumnFamily> map = new HashMap<String, ColumnFamily>();
+ for (String key : keys)
+ {
+ ColumnFamily cf = ColumnFamily.create(ksname, cfname);
+ cf.addColumn(new Column(ByteBuffer.wrap(key.getBytes()), ByteBuffer.wrap(key.getBytes()), 0));
+ map.put(key, cf);
+ }
+ return write(map);
+ }
+
+ public SSTableReader write(Map<String, ColumnFamily> entries) throws IOException
+ {
+ Map<ByteBuffer, ByteBuffer> map = new HashMap<ByteBuffer, ByteBuffer>();
+ for (Map.Entry<String, ColumnFamily> entry : entries.entrySet())
+ {
+ DataOutputBuffer buffer = new DataOutputBuffer();
+ ColumnFamily.serializer().serializeWithIndexes(entry.getValue(), buffer);
+ map.put(ByteBuffer.wrap(entry.getKey().getBytes()),
+ ByteBuffer.wrap(buffer.asByteArray()));
+ }
+ return writeRaw(map);
+ }
+
+ /**
+ * @Deprecated: Writes the binary content of a row, which should be encapsulated.
+ */
+ public SSTableReader writeRaw(Map<ByteBuffer, ByteBuffer> entries) throws IOException
+ {
+ File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, generation) : new File(dest.filenameFor(Component.DATA));
+ SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size());
+ SortedMap<DecoratedKey, ByteBuffer> sortedEntries = new TreeMap<DecoratedKey, ByteBuffer>();
+ for (Map.Entry<ByteBuffer, ByteBuffer> entry : entries.entrySet())
+ sortedEntries.put(writer.partitioner.decorateKey(entry.getKey()), entry.getValue());
+ for (Map.Entry<DecoratedKey, ByteBuffer> entry : sortedEntries.entrySet())
+ writer.append(entry.getKey(), entry.getValue());
+ SSTableReader reader = writer.closeAndOpenReader();
+ if (cleanup)
+ for (Component comp : reader.components)
+ new File(reader.descriptor.filenameFor(comp)).deleteOnExit();
+ return reader;
+ }
+ }
}
Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java?rev=1053638&r1=1053637&r2=1053638&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java Wed Dec 29 14:16:55 2010
@@ -75,7 +75,7 @@ public class SSTableWriterTest extends C
ColumnFamily.serializer().serializeWithIndexes(cf, buffer);
entries.put(ByteBufferUtil.bytes("k3"), ByteBuffer.wrap(Arrays.copyOf(buffer.getData(), buffer.getLength())));
- SSTableReader orig = SSTableUtils.writeRawSSTable("Keyspace1", "Indexed1", entries);
+ SSTableReader orig = SSTableUtils.prepare().cf("Indexed1").writeRaw(entries);
// whack the index to trigger the recover
FileUtils.deleteWithConfirm(orig.descriptor.filenameFor(Component.PRIMARY_INDEX));
FileUtils.deleteWithConfirm(orig.descriptor.filenameFor(Component.FILTER));
Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java?rev=1053638&r1=1053637&r2=1053638&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java Wed Dec 29 14:16:55 2010
@@ -119,7 +119,7 @@ public class StreamingTransferTest exten
content.add("transfer1");
content.add("transfer2");
content.add("transfer3");
- SSTableReader sstable = SSTableUtils.writeSSTable(content);
+ SSTableReader sstable = SSTableUtils.prepare().write(content);
String tablename = sstable.getTableName();
String cfname = sstable.getColumnFamilyName();
@@ -127,7 +127,7 @@ public class StreamingTransferTest exten
content2.add("test");
content2.add("test2");
content2.add("test3");
- SSTableReader sstable2 = SSTableUtils.writeSSTable(content2);
+ SSTableReader sstable2 = SSTableUtils.prepare().write(content2);
// transfer the first and last key
IPartitioner p = StorageService.getPartitioner();