You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/06/22 23:17:46 UTC
svn commit: r787404 - in /incubator/cassandra/trunk:
src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/
src/java/org/apache/cassandra/io/ test/unit/org/apache/cassandra/db/
test/unit/org/apache/cassandra/io/
Author: jbellis
Date: Mon Jun 22 21:17:46 2009
New Revision: 787404
URL: http://svn.apache.org/viewvc?rev=787404&view=rev
Log:
multitable support.
fix CF table dependencies, making CommitLogTest pass. Also fixes CASSANDRA-188 in passing.
fix tablename being null bugs by always parsing tablename from filename.
fix last part of SequenceFile.
patch by jbellis; reviewed by goffinet for CASSANDRA-79
Removed:
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/TypeInfo.java
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIndexer.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIterator.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/SystemTable.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SequenceFile.java
incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java
incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Mon Jun 22 21:17:46 2009
@@ -358,6 +358,10 @@
{
throw new ConfigurationException("invalid column sort value " + rawColumnIndexType);
}
+ if ("Super".equals(columnType) && !"Name".equals(columnIndexType))
+ {
+ throw new ConfigurationException("Super columnfamilies may only be name-sorted");
+ }
// see if flush period is set
String flushPeriodInMinutes = XMLUtils.getAttributeValue(columnFamily, "FlushPeriodInMinutes");
@@ -816,17 +820,17 @@
return dataFileDirectory;
}
- public static TypeInfo getTypeInfo(String tableName, String cfName)
+ public static ColumnComparatorFactory.ComparatorType getTypeInfo(String tableName, String cfName)
{
assert tableName != null;
CFMetaData cfMetadata = DatabaseDescriptor.getCFMetaData(tableName, cfName);
if ( cfMetadata.indexProperty_.equals("Name") )
{
- return TypeInfo.STRING;
+ return ColumnComparatorFactory.ComparatorType.NAME;
}
else
{
- return TypeInfo.LONG;
+ return ColumnComparatorFactory.ComparatorType.TIMESTAMP;
}
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java Mon Jun 22 21:17:46 2009
@@ -145,7 +145,7 @@
* Use the SSTable to write the contents of the TreeMap
* to disk.
*/
- SSTable ssTable = new SSTable(directory, filename, null, StorageService.getPartitioner());
+ SSTable ssTable = new SSTable(directory, filename, StorageService.getPartitioner());
List<String> keys = new ArrayList<String>( columnFamilies_.keySet() );
Collections.sort(keys);
/* Use this BloomFilter to decide if a key exists in a SSTable */
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java Mon Jun 22 21:17:46 2009
@@ -96,8 +96,7 @@
{
Comparator<IColumn> comparator;
String columnType = DatabaseDescriptor.getColumnFamilyType(tableName, cfName);
- if ("Super".equals(columnType)
- || DatabaseDescriptor.isNameSortingEnabled(tableName, cfName))
+ if (DatabaseDescriptor.isNameSortingEnabled(tableName, cfName))
{
comparator = ColumnComparatorFactory.getComparator(ColumnComparatorFactory.ComparatorType.NAME);
}
@@ -299,10 +298,6 @@
return table_;
}
- public void setTable_(String table_) {
- this.table_ = table_;
- }
-
/*
* This function will calculate the difference between 2 column families.
* The external input is assumed to be a superset of internal.
@@ -349,7 +344,7 @@
return columns_.getComparator();
}
- private ColumnComparatorFactory.ComparatorType getComparatorType()
+ public ColumnComparatorFactory.ComparatorType getComparatorType()
{
return getComparator() == ColumnComparatorFactory.nameComparator_
? ColumnComparatorFactory.ComparatorType.NAME
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Jun 22 21:17:46 2009
@@ -188,7 +188,7 @@
/* There are no files to compact just add to the list of SSTables */
ssTables_.addAll(filenames);
/* Load the index files and the Bloom Filters associated with them. */
- SSTable.onStart(filenames, table_);
+ SSTable.onStart(filenames);
MinorCompactionManager.instance().submit(ColumnFamilyStore.this);
if (columnFamily_.equals(Table.hints_))
{
@@ -577,7 +577,7 @@
private ColumnFamily fetchColumnFamily(String key, String cf, IFilter filter, String ssTableFile) throws IOException
{
- SSTable ssTable = new SSTable(ssTableFile, null, StorageService.getPartitioner());
+ SSTable ssTable = new SSTable(ssTableFile, StorageService.getPartitioner());
DataInputBuffer bufIn;
bufIn = filter.next(key, cf, ssTable);
if (bufIn.getLength() == 0)
@@ -1119,15 +1119,12 @@
if (ssTableRange == null)
{
String [] temp = null;
- String tableName;
- temp = rangeFileLocation.split("-");
- tableName = temp[0];
if (target != null)
{
rangeFileLocation = rangeFileLocation + System.getProperty("file.separator") + "bootstrap";
}
FileUtils.createDirectory(rangeFileLocation);
- ssTableRange = new SSTable(rangeFileLocation, mergedFileName, tableName, StorageService.getPartitioner());
+ ssTableRange = new SSTable(rangeFileLocation, mergedFileName, StorageService.getPartitioner());
}
try
{
@@ -1323,7 +1320,7 @@
if (ssTable == null)
{
- ssTable = new SSTable(compactionFileLocation, mergedFileName, null, StorageService.getPartitioner());
+ ssTable = new SSTable(compactionFileLocation, mergedFileName, StorageService.getPartitioner());
}
ssTable.append(lastkey, bufOut);
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIndexer.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIndexer.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIndexer.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIndexer.java Mon Jun 22 21:17:46 2009
@@ -57,8 +57,7 @@
dos.write(bufOut.getData(), 0, bufOut.getLength());
/* Do the indexing */
- TypeInfo typeInfo = DatabaseDescriptor.getTypeInfo(columnFamily.getTable(), columnFamily.name());
- doIndexing(typeInfo, columns, dos);
+ doIndexing(columnFamily.getComparatorType(), columns, dos);
}
/**
@@ -92,30 +91,14 @@
return bf;
}
- private static IndexHelper.ColumnIndexInfo getColumnIndexInfo(TypeInfo typeInfo, IColumn column)
+ private static IndexHelper.ColumnIndexInfo getColumnIndexInfo(ColumnComparatorFactory.ComparatorType typeInfo, IColumn column)
{
IndexHelper.ColumnIndexInfo cIndexInfo = null;
- if ( column instanceof SuperColumn )
- {
- cIndexInfo = IndexHelper.ColumnIndexFactory.instance(TypeInfo.STRING);
- cIndexInfo.set(column.name());
- }
- else
- {
- cIndexInfo = IndexHelper.ColumnIndexFactory.instance(typeInfo);
- switch(typeInfo)
- {
- case STRING:
- cIndexInfo.set(column.name());
- break;
-
- case LONG:
- cIndexInfo.set(column.timestamp());
- break;
- }
- }
-
+ cIndexInfo = IndexHelper.ColumnIndexFactory.instance(typeInfo);
+ cIndexInfo.set(typeInfo == ColumnComparatorFactory.ComparatorType.NAME
+ ? column.name() : column.timestamp());
+
return cIndexInfo;
}
@@ -124,13 +107,11 @@
* the name index is generated and written into the provided
* stream
* @param columns for whom the name index needs to be generated
- * @param bf bloom filter that summarizes the columns that make
- * up the column family.
* @param dos stream into which the serialized name index needs
* to be written.
* @throws IOException
*/
- private static void doIndexing(TypeInfo typeInfo, Collection<IColumn> columns, DataOutputStream dos) throws IOException
+ private static void doIndexing(ColumnComparatorFactory.ComparatorType typeInfo, Collection<IColumn> columns, DataOutputStream dos) throws IOException
{
/* we are going to write column indexes */
int numColumns = 0;
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIterator.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIterator.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnIterator.java Mon Jun 22 21:17:46 2009
@@ -58,7 +58,7 @@
throws IOException
{
this.isAscending = isAscending;
- SSTable ssTable = new SSTable(filename, null, StorageService.getPartitioner());
+ SSTable ssTable = new SSTable(filename, StorageService.getPartitioner());
reader = ssTable.getColumnGroupReader(key, cfName, startColumn, isAscending);
this.startColumn = startColumn;
curColumnIndex = isAscending ? 0 : -1;
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java Mon Jun 22 21:17:46 2009
@@ -272,7 +272,7 @@
for (File file : clogs)
{
// IFileReader reader = SequenceFile.bufferedReader(file.getAbsolutePath(), DatabaseDescriptor.getLogFileSizeThreshold());
- IFileReader reader = SequenceFile.reader(table_, file.getAbsolutePath());
+ IFileReader reader = SequenceFile.reader(file.getAbsolutePath());
try
{
CommitLogHeader clHeader = readCommitLogHeader(reader);
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Mon Jun 22 21:17:46 2009
@@ -253,7 +253,7 @@
String directory = DatabaseDescriptor.getDataFileLocation();
String filename = cfStore.getTempFileName();
- SSTable ssTable = new SSTable(directory, filename, table_, StorageService.getPartitioner());
+ SSTable ssTable = new SSTable(directory, filename, StorageService.getPartitioner());
// sort keys in the order they would be in when decorated
final IPartitioner partitioner = StorageService.getPartitioner();
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/SystemTable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/SystemTable.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/SystemTable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/SystemTable.java Mon Jun 22 21:17:46 2009
@@ -82,7 +82,7 @@
table_ = table;
String systemTable = getFileName();
writer_ = SequenceFile.writer(systemTable);
- reader_ = SequenceFile.reader(systemTable, table);
+ reader_ = SequenceFile.reader(systemTable);
}
private String getFileName()
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Mon Jun 22 21:17:46 2009
@@ -92,7 +92,7 @@
{
String file = getFileName(table);
writer_ = SequenceFile.writer(file);
- reader_ = SequenceFile.reader(file, table);
+ reader_ = SequenceFile.reader(file);
Table.TableMetadata.load(table);
metadata = new Table.TableMetadata();
@@ -118,7 +118,7 @@
if ( reader_ == null )
{
- reader_ = SequenceFile.reader(file, table);
+ reader_ = SequenceFile.reader(file);
}
while ( !reader_.isEOF() )
@@ -962,7 +962,7 @@
// sstables
for (String filename : cfs.getSSTableFilenames())
{
- FileStruct fs = new FileStruct(SequenceFile.reader(filename, table_), StorageService.getPartitioner());
+ FileStruct fs = new FileStruct(SequenceFile.reader(filename), StorageService.getPartitioner());
fs.seekTo(startWith);
iterators.add(fs);
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java Mon Jun 22 21:17:46 2009
@@ -26,7 +26,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.IColumn;
-import org.apache.cassandra.db.TypeInfo;
+import org.apache.cassandra.db.ColumnComparatorFactory;
import org.apache.cassandra.utils.FBUtilities;
@@ -152,12 +152,8 @@
DataInputBuffer indexIn = new DataInputBuffer();
indexIn.reset(indexOut.getData(), indexOut.getLength());
- TypeInfo typeInfo = DatabaseDescriptor.getTypeInfo(tableName, cfName);
- if ( DatabaseDescriptor.getColumnFamilyType(tableName, cfName).equals("Super") || DatabaseDescriptor.isNameSortingEnabled(tableName, cfName) )
- {
- typeInfo = TypeInfo.STRING;
- }
-
+ ColumnComparatorFactory.ComparatorType typeInfo = DatabaseDescriptor.getTypeInfo(tableName, cfName);
+
while(indexIn.available() > 0)
{
ColumnIndexInfo cIndexInfo = ColumnIndexFactory.instance(typeInfo);
@@ -281,7 +277,7 @@
* binary search.
*/
Comparator<IndexHelper.ColumnIndexInfo> comparator = Collections.reverseOrder();
- IndexHelper.ColumnIndexInfo rhs = IndexHelper.ColumnIndexFactory.instance(TypeInfo.LONG);
+ IndexHelper.ColumnIndexInfo rhs = IndexHelper.ColumnIndexFactory.instance(ColumnComparatorFactory.ComparatorType.TIMESTAMP);
rhs.set(timeRange.rhs());
int index = Collections.binarySearch(columnIndexList, rhs, comparator);
if ( index < 0 )
@@ -307,7 +303,7 @@
{
int chunks = columnIndexList.size();
/* Index info for the lower bound of the time range */
- IndexHelper.ColumnIndexInfo lhs = IndexHelper.ColumnIndexFactory.instance(TypeInfo.LONG);
+ IndexHelper.ColumnIndexInfo lhs = IndexHelper.ColumnIndexFactory.instance(ColumnComparatorFactory.ComparatorType.TIMESTAMP);
lhs.set(timeRange.lhs());
int i = index + 1;
for ( ; i < chunks; ++i )
@@ -339,21 +335,11 @@
public static class ColumnIndexFactory
{
- public static ColumnIndexInfo instance(TypeInfo typeInfo)
+ public static ColumnIndexInfo instance(ColumnComparatorFactory.ComparatorType typeInfo)
{
- ColumnIndexInfo cIndexInfo = null;
- switch(typeInfo)
- {
- case STRING:
- cIndexInfo = new ColumnNameIndexInfo();
- break;
-
- case LONG:
- cIndexInfo = new ColumnTimestampIndexInfo();
- break;
- }
- return cIndexInfo;
- }
+ return typeInfo == ColumnComparatorFactory.ComparatorType.NAME
+ ? new ColumnNameIndexInfo() : new ColumnTimestampIndexInfo();
+ }
}
/**
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java Mon Jun 22 21:17:46 2009
@@ -150,13 +150,13 @@
* associated with these files. Also caches the file handles
* associated with these files.
*/
- public static void onStart(List<String> filenames, String tableName) throws IOException
+ public static void onStart(List<String> filenames) throws IOException
{
for (String filename : filenames)
{
try
{
- new SSTable(filename, tableName, StorageService.getPartitioner());
+ new SSTable(filename, StorageService.getPartitioner());
}
catch (IOException ex)
{
@@ -205,17 +205,15 @@
private String lastWrittenKey_;
private IPartitioner partitioner_;
- private String table_;
/**
* This ctor basically gets passed in the full path name
* of the data file associated with this SSTable. Use this
* ctor to read the data in this file.
*/
- public SSTable(String dataFileName, String tableName, IPartitioner partitioner) throws IOException
+ public SSTable(String dataFileName, IPartitioner partitioner) throws IOException
{
dataFile_ = dataFileName;
partitioner_ = partitioner;
- table_ = tableName;
/*
* this is to prevent multiple threads from
* loading the same index files multiple times
@@ -237,15 +235,20 @@
* This ctor is used for writing data into the SSTable. Use this
* version for non DB writes to the SSTable.
*/
- public SSTable(String directory, String filename, String tableName, IPartitioner partitioner) throws IOException
+ public SSTable(String directory, String filename, IPartitioner partitioner) throws IOException
{
dataFile_ = directory + System.getProperty("file.separator") + filename + "-Data.db";
partitioner_ = partitioner;
dataWriter_ = SequenceFile.bufferedWriter(dataFile_, 4 * 1024 * 1024);
- table_ = tableName;
indexRAF_ = new BufferedRandomAccessFile(indexFilename(), "rw", 1024 * 1024);
}
+ static String parseTableName(String filename)
+ {
+ String[] parts = new File(filename).getName().split("-"); // table, cf, index, [filetype]
+ return parts[0];
+ }
+
private void loadBloomFilter() throws IOException
{
assert bfs_.get(dataFile_) == null;
@@ -477,7 +480,7 @@
IFileReader dataReader = null;
try
{
- dataReader = SequenceFile.reader(dataFile_, table_);
+ dataReader = SequenceFile.reader(dataFile_);
String decoratedKey = partitioner_.decorateKey(clientKey);
long position = getPosition(decoratedKey, dataReader, partitioner_);
@@ -559,22 +562,19 @@
*/
public ColumnGroupReader getColumnGroupReader(String key, String cfName, String startColumn, boolean isAscending) throws IOException
{
- ColumnGroupReader reader = null;
- IFileReader dataReader = SequenceFile.reader(table_, dataFile_);
+ IFileReader dataReader = SequenceFile.reader(dataFile_);
try
{
/* Morph key into actual key based on the partition type. */
String decoratedKey = partitioner_.decorateKey(key);
long position = getPosition(decoratedKey, dataReader, partitioner_);
- reader = new ColumnGroupReader(dataFile_, decoratedKey, cfName, startColumn, isAscending, position);
+ return new ColumnGroupReader(dataFile_, decoratedKey, cfName, startColumn, isAscending, position);
}
finally
{
- if (dataReader != null)
- dataReader.close();
+ dataReader.close();
}
- return reader;
}
/** obviously only for testing */
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SequenceFile.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SequenceFile.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SequenceFile.java Mon Jun 22 21:17:46 2009
@@ -516,6 +516,8 @@
{
private String key_;
private String cfName_;
+ private String cfType_;
+ private int indexType_;
private boolean isAscending_;
private List<IndexHelper.ColumnIndexInfo> columnIndexList_;
@@ -585,16 +587,18 @@
/* read the index */
List<IndexHelper.ColumnIndexInfo> colIndexList = new ArrayList<IndexHelper.ColumnIndexInfo>();
if (hasColumnIndexes)
- totalBytesRead += IndexHelper.deserializeIndex(null, cfName_, file_, colIndexList);
+ totalBytesRead += IndexHelper.deserializeIndex(getTableName(), cfName_, file_, colIndexList);
/* need to do two things here.
* 1. move the file pointer to the beginning of the list of stored columns
* 2. calculate the size of all columns */
String cfName = file_.readUTF();
+ cfType_ = file_.readUTF();
+ indexType_ = file_.readInt();
localDeletionTime_ = file_.readInt();
markedForDeleteAt_ = file_.readLong();
int totalNumCols = file_.readInt();
- allColumnsSize_ = dataSize - (totalBytesRead + utfPrefix_ + cfName.length() + 4 + 8 + 4);
+ allColumnsSize_ = dataSize - (totalBytesRead + 2 * utfPrefix_ + cfName.length() + cfType_.length() + 4 + 4 + 8 + 4);
columnStartPosition_ = file_.getFilePointer();
columnIndexList_ = getFullColumnIndexList(colIndexList, totalNumCols);
@@ -627,6 +631,8 @@
bufOut.reset();
// write CF info
bufOut.writeUTF(cfName_);
+ bufOut.writeUTF(cfType_);
+ bufOut.writeInt(indexType_);
bufOut.writeInt(localDeletionTime_);
bufOut.writeLong(markedForDeleteAt_);
// now write the columns
@@ -651,12 +657,15 @@
private static final short utfPrefix_ = 2;
protected RandomAccessFile file_;
protected String filename_;
- private String table_;
- AbstractReader(String filename, String tableName)
+ AbstractReader(String filename)
{
filename_ = filename;
- table_ = tableName;
+ }
+
+ String getTableName()
+ {
+ return SSTable.parseTableName(filename_);
}
public String getFileName()
@@ -703,10 +712,11 @@
/* if we do then deserialize the index */
if (hasColumnIndexes)
{
- if (DatabaseDescriptor.isNameSortingEnabled(table_, cfName) || DatabaseDescriptor.getColumnFamilyType(table_, cfName).equals("Super"))
+ String tableName = getTableName();
+ if (DatabaseDescriptor.isNameSortingEnabled(tableName, cfName))
{
/* read the index */
- totalBytesRead += IndexHelper.deserializeIndex(table_, cfName, file_, columnIndexList);
+ totalBytesRead += IndexHelper.deserializeIndex(tableName, cfName, file_, columnIndexList);
}
else
{
@@ -734,7 +744,7 @@
if (DatabaseDescriptor.isTimeSortingEnabled(null, cfName))
{
/* read the index */
- totalBytesRead += IndexHelper.deserializeIndex(table_, cfName, file_, columnIndexList);
+ totalBytesRead += IndexHelper.deserializeIndex(getTableName(), cfName, file_, columnIndexList);
}
else
{
@@ -832,6 +842,12 @@
String cfName = file_.readUTF();
dataSize -= (utfPrefix_ + cfName.length());
+ String cfType = file_.readUTF();
+ dataSize -= (utfPrefix_ + cfType.length());
+
+ int indexType = file_.readInt();
+ dataSize -= 4;
+
/* read local deletion time */
int localDeletionTime = file_.readInt();
dataSize -=4;
@@ -852,19 +868,13 @@
file_.skipBytes((int) coordinate.start_);
dataSize = (int) (coordinate.end_ - coordinate.start_);
- /*
- * write the number of columns in the column family we are returning:
- * dataSize that we are reading +
- * length of column family name +
- * one booleanfor deleted or not +
- * one int for number of columns
- */
- bufOut.writeInt(dataSize + utfPrefix_ + cfName.length() + 4 + 8 + 4);
- /* write the column family name */
+ // returned data size
+ bufOut.writeInt(dataSize + utfPrefix_ * 2 + cfName.length() + cfType.length() + 4 + 4 + 8 + 4);
+ // echo back the CF data we read
bufOut.writeUTF(cfName);
- /* write local deletion time */
+ bufOut.writeUTF(cfType);
+ bufOut.writeInt(indexType);
bufOut.writeInt(localDeletionTime);
- /* write if this cf is marked for delete */
bufOut.writeLong(markedForDeleteAt);
/* write number of columns */
bufOut.writeInt(columnRange.count());
@@ -911,6 +921,12 @@
String cfName = file_.readUTF();
dataSize -= (utfPrefix_ + cfName.length());
+ String cfType = file_.readUTF();
+ dataSize -= (utfPrefix_ + cfType.length());
+
+ int indexType = file_.readInt();
+ dataSize -= 4;
+
/* read local deletion time */
int localDeletionTime = file_.readInt();
dataSize -=4;
@@ -940,19 +956,13 @@
dataSizeReturned += coordinate.end_ - coordinate.start_;
}
- /*
- * write the number of columns in the column family we are returning:
- * dataSize that we are reading +
- * length of column family name +
- * one booleanfor deleted or not +
- * one int for number of columns
- */
- bufOut.writeInt(dataSizeReturned + utfPrefix_ + cfName.length() + 4 + 8 + 4);
- /* write the column family name */
+ // returned data size
+ bufOut.writeInt(dataSizeReturned + utfPrefix_ * 2 + cfName.length() + cfType.length() + 4 + 4 + 8 + 4);
+ // echo back the CF data we read
bufOut.writeUTF(cfName);
- /* write local deletion time */
+ bufOut.writeUTF(cfType);
+ bufOut.writeInt(indexType);
bufOut.writeInt(localDeletionTime);
- /* write if this cf is marked for delete */
bufOut.writeLong(markedForDeleteAt);
/* write number of columns */
bufOut.writeInt(numColsReturned);
@@ -1012,9 +1022,9 @@
public static class Reader extends AbstractReader
{
- Reader(String filename, String tableName) throws IOException
+ Reader(String filename) throws IOException
{
- super(filename, tableName);
+ super(filename);
init(filename);
}
@@ -1077,7 +1087,7 @@
BufferReader(String filename, int size) throws IOException
{
- super(filename, null);
+ super(filename);
size_ = size;
}
@@ -1106,9 +1116,9 @@
return new FastConcurrentWriter(filename, size);
}
- public static IFileReader reader(String filename, String tableName) throws IOException
+ public static IFileReader reader(String filename) throws IOException
{
- return new Reader(filename, tableName);
+ return new Reader(filename);
}
public static IFileReader bufferedReader(String filename, int size) throws IOException
Modified: incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java (original)
+++ incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/TableTest.java Mon Jun 22 21:17:46 2009
@@ -37,6 +37,19 @@
private static final String TEST_KEY = "key1";
private static final String TABLE_NAME = "Table1";
+ interface Runner
+ {
+ public void run() throws Exception;
+ }
+
+ private void reTest(Runner setup, ColumnFamilyStore cfs, Runner verify) throws Exception
+ {
+ setup.run();
+ verify.run();
+ cfs.forceBlockingFlush();
+ verify.run();
+ }
+
@Test
public void testOpen() throws Throwable {
Table table = Table.open("Mailbox");
@@ -47,40 +60,66 @@
@Test
public void testGetRowSingleColumn() throws Throwable
{
- Table table = Table.open(TABLE_NAME);
- RowMutation rm = makeSimpleRowMutation();
- rm.apply();
- Row result = table.getRow(TEST_KEY, "Standard1:col1");
- ColumnFamily cres = result.getColumnFamily("Standard1");
- assertNotNull(cres);
- assertEquals(1, cres.getColumnCount());
- assertNotNull(cres.getColumn("col1"));
+ final Table table = Table.open(TABLE_NAME);
+ Runner setup = new Runner()
+ {
+ public void run() throws Exception
+ {
+ RowMutation rm = makeSimpleRowMutation();
+ rm.apply();
+ }
+ };
+ Runner verify = new Runner()
+ {
+ public void run() throws Exception
+ {
+ Row result;
+
+ result = table.getRow(TEST_KEY, "Standard1:col1");
+ assertColumns(result.getColumnFamily("Standard1"), "col1");
+
+ result = table.getRow(TEST_KEY, "Standard1:col3");
+ assertColumns(result.getColumnFamily("Standard1"), "col3");
+ }
+ };
+ reTest(setup, table.getColumnFamilyStore("Standard1"), verify);
}
@Test
public void testGetRowOffsetCount() throws Throwable
{
- Table table = Table.open(TABLE_NAME);
-
- RowMutation rm = makeSimpleRowMutation(); //inserts col1, col2, col3
+ final Table table = Table.open(TABLE_NAME);
-
- rm.apply();
- Row result = table.getRow(TEST_KEY, "Standard1", 0, 2);
- ColumnFamily cres = result.getColumnFamily("Standard1");
- assertNotNull(cres);
- assertEquals(cres.getColumnCount(), 2);
- // should have col1 and col2
- assertNotNull(cres.getColumn("col1"));
- assertNotNull(cres.getColumn("col2"));
-
- result = table.getRow(TEST_KEY, "Standard1", 1, 2);
- cres = result.getColumnFamily("Standard1");
- assertNotNull(cres);
- assertEquals(2, cres.getColumnCount());
- // offset is 1, so we should have col2 and col3
- assertNotNull(cres.getColumn("col2"));
- assertNotNull(cres.getColumn("col3"));
+ Runner setup = new Runner()
+ {
+ public void run() throws Exception
+ {
+ RowMutation rm = makeSimpleRowMutation(); //inserts col1, col2, col3
+ rm.apply();
+ }
+ };
+ Runner verify = new Runner()
+ {
+ public void run() throws Exception
+ {
+ Row result = table.getRow(TEST_KEY, "Standard1", 0, 2);
+ ColumnFamily cres = result.getColumnFamily("Standard1");
+ assertNotNull(cres);
+ assertEquals(cres.getColumnCount(), 2);
+ // should have col1 and col2
+ assertNotNull(cres.getColumn("col1"));
+ assertNotNull(cres.getColumn("col2"));
+
+ result = table.getRow(TEST_KEY, "Standard1", 1, 2);
+ cres = result.getColumnFamily("Standard1");
+ assertNotNull(cres);
+ assertEquals(2, cres.getColumnCount());
+ // offset is 1, so we should have col2 and col3
+ assertNotNull(cres.getColumn("col2"));
+ assertNotNull(cres.getColumn("col3"));
+ }
+ };
+ reTest(setup, table.getColumnFamilyStore("Standard1"), verify);
}
@Test
@@ -251,57 +290,114 @@
public void testGetSliceFromBasic() throws Throwable
{
// tests slicing against data from one row in a memtable and then flushed to an sstable
- Table table = Table.open(TABLE_NAME);
- String ROW = "row1";
- RowMutation rm = new RowMutation(TABLE_NAME, ROW);
- ColumnFamily cf = ColumnFamily.create("Table1", "Standard1");
- cf.addColumn(new Column("col1", "val1".getBytes(), 1L));
- cf.addColumn(new Column("col3", "val3".getBytes(), 1L));
- cf.addColumn(new Column("col4", "val4".getBytes(), 1L));
- cf.addColumn(new Column("col5", "val5".getBytes(), 1L));
- cf.addColumn(new Column("col7", "val7".getBytes(), 1L));
- cf.addColumn(new Column("col9", "val9".getBytes(), 1L));
- rm.add(cf);
- rm.apply();
-
- rm = new RowMutation(TABLE_NAME, ROW);
- rm.delete("Standard1:col4", 2L);
- rm.apply();
+ final Table table = Table.open(TABLE_NAME);
+ final String ROW = "row1";
+ Runner setup = new Runner()
+ {
+ public void run() throws Exception
+ {
+ RowMutation rm = new RowMutation(TABLE_NAME, ROW);
+ ColumnFamily cf = ColumnFamily.create("Table1", "Standard1");
+ cf.addColumn(new Column("col1", "val1".getBytes(), 1L));
+ cf.addColumn(new Column("col3", "val3".getBytes(), 1L));
+ cf.addColumn(new Column("col4", "val4".getBytes(), 1L));
+ cf.addColumn(new Column("col5", "val5".getBytes(), 1L));
+ cf.addColumn(new Column("col7", "val7".getBytes(), 1L));
+ cf.addColumn(new Column("col9", "val9".getBytes(), 1L));
+ rm.add(cf);
+ rm.apply();
+
+ rm = new RowMutation(TABLE_NAME, ROW);
+ rm.delete("Standard1:col4", 2L);
+ rm.apply();
+ }
+ };
- validateGetSliceFromBasic(table, ROW);
- table.getColumnFamilyStore("Standard1").forceBlockingFlush();
- validateGetSliceFromBasic(table, ROW);
+ Runner verify = new Runner()
+ {
+ public void run() throws Exception
+ {
+ Row result;
+ ColumnFamily cf;
+
+ result = table.getSliceFrom(ROW, "Standard1:col5", true, 2);
+ cf = result.getColumnFamily("Standard1");
+ assertColumns(cf, "col5", "col7");
+
+ result = table.getSliceFrom(ROW, "Standard1:col4", true, 2);
+ cf = result.getColumnFamily("Standard1");
+ assertColumns(cf, "col4", "col5", "col7");
+
+ result = table.getSliceFrom(ROW, "Standard1:col5", false, 2);
+ cf = result.getColumnFamily("Standard1");
+ assertColumns(cf, "col3", "col4", "col5");
+
+ result = table.getSliceFrom(ROW, "Standard1:col6", false, 2);
+ cf = result.getColumnFamily("Standard1");
+ assertColumns(cf, "col3", "col4", "col5");
+
+ result = table.getSliceFrom(ROW, "Standard1:col95", true, 2);
+ cf = result.getColumnFamily("Standard1");
+ assertColumns(cf);
+
+ result = table.getSliceFrom(ROW, "Standard1:col0", false, 2);
+ cf = result.getColumnFamily("Standard1");
+ assertColumns(cf);
+ }
+ };
+
+ reTest(setup, table.getColumnFamilyStore("Standard1"), verify);
}
@Test
public void testGetSliceFromAdvanced() throws Throwable
{
// tests slicing against data from one row spread across two sstables
- Table table = Table.open(TABLE_NAME);
- String ROW = "row2";
- RowMutation rm = new RowMutation(TABLE_NAME, ROW);
- ColumnFamily cf = ColumnFamily.create("Table1", "Standard1");
- cf.addColumn(new Column("col1", "val1".getBytes(), 1L));
- cf.addColumn(new Column("col2", "val2".getBytes(), 1L));
- cf.addColumn(new Column("col3", "val3".getBytes(), 1L));
- cf.addColumn(new Column("col4", "val4".getBytes(), 1L));
- cf.addColumn(new Column("col5", "val5".getBytes(), 1L));
- cf.addColumn(new Column("col6", "val6".getBytes(), 1L));
- rm.add(cf);
- rm.apply();
- table.getColumnFamilyStore("Standard1").forceBlockingFlush();
-
- rm = new RowMutation(TABLE_NAME, ROW);
- cf = ColumnFamily.create("Table1", "Standard1");
- cf.addColumn(new Column("col1", "valx".getBytes(), 2L));
- cf.addColumn(new Column("col2", "valx".getBytes(), 2L));
- cf.addColumn(new Column("col3", "valx".getBytes(), 2L));
- rm.add(cf);
- rm.apply();
+ final Table table = Table.open(TABLE_NAME);
+ final String ROW = "row2";
+ Runner setup = new Runner()
+ {
+ public void run() throws Exception
+ {
+ RowMutation rm = new RowMutation(TABLE_NAME, ROW);
+ ColumnFamily cf = ColumnFamily.create("Table1", "Standard1");
+ cf.addColumn(new Column("col1", "val1".getBytes(), 1L));
+ cf.addColumn(new Column("col2", "val2".getBytes(), 1L));
+ cf.addColumn(new Column("col3", "val3".getBytes(), 1L));
+ cf.addColumn(new Column("col4", "val4".getBytes(), 1L));
+ cf.addColumn(new Column("col5", "val5".getBytes(), 1L));
+ cf.addColumn(new Column("col6", "val6".getBytes(), 1L));
+ rm.add(cf);
+ rm.apply();
+ table.getColumnFamilyStore("Standard1").forceBlockingFlush();
+
+ rm = new RowMutation(TABLE_NAME, ROW);
+ cf = ColumnFamily.create("Table1", "Standard1");
+ cf.addColumn(new Column("col1", "valx".getBytes(), 2L));
+ cf.addColumn(new Column("col2", "valx".getBytes(), 2L));
+ cf.addColumn(new Column("col3", "valx".getBytes(), 2L));
+ rm.add(cf);
+ rm.apply();
+ }
+ };
- validateGetSliceFromAdvanced(table, ROW);
- table.getColumnFamilyStore("Standard1").forceBlockingFlush();
- validateGetSliceFromAdvanced(table, ROW);
+ Runner verify = new Runner()
+ {
+ public void run() throws Exception
+ {
+ Row result;
+ ColumnFamily cfres;
+
+ result = table.getSliceFrom(ROW, "Standard1:col2", true, 3);
+ cfres = result.getColumnFamily("Standard1");
+ assertColumns(cfres, "col2", "col3", "col4");
+ assertEquals(new String(cfres.getColumn("col2").value()), "valx");
+ assertEquals(new String(cfres.getColumn("col3").value()), "valx");
+ assertEquals(new String(cfres.getColumn("col4").value()), "val4");
+ }
+ };
+
+ reTest(setup, table.getColumnFamilyStore("Standard1"), verify);
}
@Test
@@ -362,46 +458,4 @@
: "Columns [" + StringUtils.join(columns, ", ") + "] is not expected [" + StringUtils.join(columnNames, ", ") + "]";
}
- private void validateGetSliceFromAdvanced(Table table, String row) throws Throwable
- {
- Row result;
- ColumnFamily cfres;
-
- result = table.getSliceFrom(row, "Standard1:col2", true, 3);
- cfres = result.getColumnFamily("Standard1");
- assertColumns(cfres, "col2", "col3", "col4");
- assertEquals(new String(cfres.getColumn("col2").value()), "valx");
- assertEquals(new String(cfres.getColumn("col3").value()), "valx");
- assertEquals(new String(cfres.getColumn("col4").value()), "val4");
- }
-
- private void validateGetSliceFromBasic(Table table, String row) throws Throwable
- {
- Row result;
- ColumnFamily cf;
-
- result = table.getSliceFrom(row, "Standard1:col5", true, 2);
- cf = result.getColumnFamily("Standard1");
- assertColumns(cf, "col5", "col7");
-
- result = table.getSliceFrom(row, "Standard1:col4", true, 2);
- cf = result.getColumnFamily("Standard1");
- assertColumns(cf, "col4", "col5", "col7");
-
- result = table.getSliceFrom(row, "Standard1:col5", false, 2);
- cf = result.getColumnFamily("Standard1");
- assertColumns(cf, "col3", "col4", "col5");
-
- result = table.getSliceFrom(row, "Standard1:col6", false, 2);
- cf = result.getColumnFamily("Standard1");
- assertColumns(cf, "col3", "col4", "col5");
-
- result = table.getSliceFrom(row, "Standard1:col95", true, 2);
- cf = result.getColumnFamily("Standard1");
- assertColumns(cf);
-
- result = table.getSliceFrom(row, "Standard1:col0", false, 2);
- cf = result.getColumnFamily("Standard1");
- assertColumns(cf);
- }
}
Modified: incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java?rev=787404&r1=787403&r2=787404&view=diff
==============================================================================
--- incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java (original)
+++ incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java Mon Jun 22 21:17:46 2009
@@ -56,7 +56,7 @@
private void verifySingle(File f, byte[] bytes, String key) throws IOException
{
- SSTable ssTable = new SSTable(f.getPath() + "-Data.db", "Table1", new OrderPreservingPartitioner());
+ SSTable ssTable = new SSTable(f.getPath() + "-Data.db", new OrderPreservingPartitioner());
FileStruct fs = new FileStruct(SequenceFile.bufferedReader(ssTable.dataFile_, 128 * 1024), new OrderPreservingPartitioner());
fs.seekTo(key);
int size = fs.getBufIn().readInt();
@@ -95,7 +95,7 @@
{
List<String> keys = new ArrayList(map.keySet());
Collections.shuffle(keys);
- SSTable ssTable = new SSTable(f.getPath() + "-Data.db", "Table1", new OrderPreservingPartitioner());
+ SSTable ssTable = new SSTable(f.getPath() + "-Data.db", new OrderPreservingPartitioner());
FileStruct fs = new FileStruct(SequenceFile.bufferedReader(ssTable.dataFile_, 128 * 1024), new OrderPreservingPartitioner());
for (String key : keys)
{