You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/09/02 17:04:16 UTC
svn commit: r810540 - in /incubator/cassandra/trunk: ./ conf/ lib/
lib/licenses/ src/java/org/apache/cassandra/config/
src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/
test/unit/org/apache/cassandra/io/
Author: jbellis
Date: Wed Sep 2 15:04:16 2009
New Revision: 810540
URL: http://svn.apache.org/viewvc?rev=810540&view=rev
Log:
remove buggy concurrentlinkedhashmap library and lru cache. too late in 0.4 to try to debug the library -- will revisit for 0.5. patch by jbellis; reviewed by Chris Goffinet for CASSANDRA-405
Removed:
incubator/cassandra/trunk/lib/clhm-20090629.jar
incubator/cassandra/trunk/lib/licenses/clhm-20090629.jar.LICENSE
Modified:
incubator/cassandra/trunk/CHANGES.txt
incubator/cassandra/trunk/conf/storage-conf.xml
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableWriter.java
incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java
Modified: incubator/cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/CHANGES.txt?rev=810540&r1=810539&r2=810540&view=diff
==============================================================================
--- incubator/cassandra/trunk/CHANGES.txt (original)
+++ incubator/cassandra/trunk/CHANGES.txt Wed Sep 2 15:04:16 2009
@@ -6,7 +6,6 @@
* Scan all sstables for all queries to avoid situations where
different types of operation on the same ColumnFamily could
disagree on what data was present
- * Configurable LRU cache for key lookups
* Snapshot support via JMX
* BinaryMemtable support for bulk load via hadoop; see
https://github.com/lenn0x/Cassandra-Hadoop-BMT/tree
Modified: incubator/cassandra/trunk/conf/storage-conf.xml
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/conf/storage-conf.xml?rev=810540&r1=810539&r2=810540&view=diff
==============================================================================
--- incubator/cassandra/trunk/conf/storage-conf.xml (original)
+++ incubator/cassandra/trunk/conf/storage-conf.xml Wed Sep 2 15:04:16 2009
@@ -42,17 +42,6 @@
-->
<Keyspaces>
<Keyspace Name="Keyspace1">
- <!--
- ~ The fraction of keys per sstable whose locations we keep in
- ~ memory in "mostly LRU" order. (JUST the key locations, NOT any
- ~ column values.)
-
- ~ The amount of memory used by the default setting of 0.01 is
- ~ comparable to the amount used by the internal per-sstable key
- ~ index. Consider increasing this if you have fewer, wider rows.
- ~ Set to 0 to disable entirely.
- -->
- <KeysCachedFraction>0.01</KeysCachedFraction>
<!--
~ The CompareWith attribute tells Cassandra how to sort the columns
~ for slicing operations. The default is BytesType, which is a
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=810540&r1=810539&r2=810540&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Wed Sep 2 15:04:16 2009
@@ -89,7 +89,6 @@
private static String d_columnValue_ = "COLUMN_VALUE";
private static String d_columnTimestamp_ = "COLUMN_TIMESTAMP";
- private static Map<String, Double> tableKeysCachedFractions_;
/*
* A map from table names to the set of column families for the table and the
* corresponding meta data for that column family.
@@ -401,7 +400,6 @@
CommitLog.setSegmentSize(Integer.parseInt(value) * 1024 * 1024);
tableToCFMetaDataMap_ = new HashMap<String, Map<String, CFMetaData>>();
- tableKeysCachedFractions_ = new HashMap<String, Double>();
/* See which replica placement strategy to use */
String replicaPlacementStrategyClassName = xmlUtils.getNodeValue("/Storage/ReplicaPlacementStrategy");
@@ -438,17 +436,6 @@
tables_.add(tName);
tableToCFMetaDataMap_.put(tName, new HashMap<String, CFMetaData>());
- String xqlCacheSize = "/Storage/Keyspaces/Keyspace[@Name='" + tName + "']/KeysCachedFraction";
- value = xmlUtils.getNodeValue(xqlCacheSize);
- if (value == null)
- {
- tableKeysCachedFractions_.put(tName, 0.01);
- }
- else
- {
- tableKeysCachedFractions_.put(tName, Double.valueOf(value));
- }
-
String xqlTable = "/Storage/Keyspaces/Keyspace[@Name='" + tName + "']/";
NodeList columnFamilies = xmlUtils.getRequestedNodeList(xqlTable + "ColumnFamily");
@@ -573,7 +560,6 @@
data.subcolumnComparator = new BytesType();
systemMetadata.put(HintedHandOffManager.HINTS_CF, data);
tableToCFMetaDataMap_.put(Table.SYSTEM_TABLE, systemMetadata);
- tableKeysCachedFractions_.put(Table.SYSTEM_TABLE, 0.01);
/* make sure we have a directory for each table */
createTableDirectories();
@@ -964,11 +950,6 @@
return tableToCFMetaDataMap_;
}
- public static double getKeysCachedFraction(String tableName)
- {
- return tableKeysCachedFractions_.get(tableName);
- }
-
private static class ConfigurationException extends Exception
{
public ConfigurationException(String message)
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java?rev=810540&r1=810539&r2=810540&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java Wed Sep 2 15:04:16 2009
@@ -175,7 +175,7 @@
writer.append(key, bytes);
}
}
- cfStore.storeLocation(writer.closeAndOpenReader(DatabaseDescriptor.getKeysCachedFraction(table_)));
+ cfStore.storeLocation(writer.closeAndOpenReader());
columnFamilies_.clear();
}
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=810540&r1=810539&r2=810540&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Wed Sep 2 15:04:16 2009
@@ -1009,7 +1009,7 @@
if (rangeWriter != null)
{
- rangeWriter.closeAndOpenReader(DatabaseDescriptor.getKeysCachedFraction(table_));
+ rangeWriter.closeAndOpenReader();
if (fileList != null)
{
//Retain order. The -Data.db file needs to be last because
@@ -1160,7 +1160,7 @@
if (writer != null)
{
// TODO if all the keys were the same nothing will be done here
- ssTable = writer.closeAndOpenReader(DatabaseDescriptor.getKeysCachedFraction(table_));
+ ssTable = writer.closeAndOpenReader();
newfile = writer.getFilename();
}
sstableLock_.writeLock().lock();
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=810540&r1=810539&r2=810540&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Wed Sep 2 15:04:16 2009
@@ -212,7 +212,7 @@
writer.append(partitioner.decorateKey(key), buffer);
}
}
- SSTableReader ssTable = writer.closeAndOpenReader(DatabaseDescriptor.getKeysCachedFraction(table_));
+ SSTableReader ssTable = writer.closeAndOpenReader();
cfStore.onMemtableFlush(cLogCtx);
cfStore.storeLocation(ssTable);
buffer.close();
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java?rev=810540&r1=810539&r2=810540&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java Wed Sep 2 15:04:16 2009
@@ -31,7 +31,6 @@
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.marshal.AbstractType;
import org.cliffc.high_scale_lib.NonBlockingHashMap;
-import com.reardencommerce.kernel.collections.shared.evictable.ConcurrentLinkedHashMap;
public class SSTableReader extends SSTable
{
@@ -83,10 +82,10 @@
public static synchronized SSTableReader open(String dataFileName) throws IOException
{
- return open(dataFileName, StorageService.getPartitioner(), DatabaseDescriptor.getKeysCachedFraction(parseTableName(dataFileName)));
+ return open(dataFileName, StorageService.getPartitioner());
}
- public static synchronized SSTableReader open(String dataFileName, IPartitioner partitioner, double cacheFraction) throws IOException
+ public static synchronized SSTableReader open(String dataFileName, IPartitioner partitioner) throws IOException
{
SSTableReader sstable = openedFiles.get(dataFileName);
if (sstable == null)
@@ -97,10 +96,6 @@
long start = System.currentTimeMillis();
sstable.loadIndexFile();
sstable.loadBloomFilter();
- if (cacheFraction > 0)
- {
- sstable.keyCache = createKeyCache((int)((sstable.getIndexPositions().size() + 1) * INDEX_INTERVAL * cacheFraction));
- }
if (logger.isDebugEnabled())
logger.debug("INDEX LOAD TIME for " + dataFileName + ": " + (System.currentTimeMillis() - start) + " ms.");
@@ -116,20 +111,11 @@
return sstable;
}
- public static ConcurrentLinkedHashMap<String, Long> createKeyCache(int size)
- {
- return ConcurrentLinkedHashMap.create(ConcurrentLinkedHashMap.EvictionPolicy.SECOND_CHANCE, size);
- }
-
-
- private ConcurrentLinkedHashMap<String, Long> keyCache;
-
- SSTableReader(String filename, IPartitioner partitioner, List<KeyPosition> indexPositions, BloomFilter bloomFilter, ConcurrentLinkedHashMap<String, Long> keyCache)
+ SSTableReader(String filename, IPartitioner partitioner, List<KeyPosition> indexPositions, BloomFilter bloomFilter)
{
super(filename, partitioner);
this.indexPositions = indexPositions;
this.bf = bloomFilter;
- this.keyCache = keyCache;
synchronized (SSTableReader.this)
{
openedFiles.put(filename, this);
@@ -202,14 +188,6 @@
{
if (!bf.isPresent(decoratedKey))
return -1;
- if (keyCache != null)
- {
- Long cachedPosition = keyCache.get(decoratedKey);
- if (cachedPosition != null)
- {
- return cachedPosition;
- }
- }
long start = getIndexScanPosition(decoratedKey, partitioner);
if (start < 0)
{
@@ -237,8 +215,6 @@
int v = partitioner.getDecoratedKeyComparator().compare(indexDecoratedKey, decoratedKey);
if (v == 0)
{
- if (keyCache != null)
- keyCache.put(decoratedKey, position);
return position;
}
if (v > 0)
@@ -307,7 +283,7 @@
openedFiles.clear();
for (SSTableReader sstable : sstables)
{
- SSTableReader.open(sstable.path, sstable.partitioner, 0.01);
+ SSTableReader.open(sstable.path, sstable.partitioner);
}
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableWriter.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableWriter.java?rev=810540&r1=810539&r2=810540&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableWriter.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableWriter.java Wed Sep 2 15:04:16 2009
@@ -34,7 +34,6 @@
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.BloomFilter;
import org.apache.cassandra.config.DatabaseDescriptor;
-import com.reardencommerce.kernel.collections.shared.evictable.ConcurrentLinkedHashMap;
public class SSTableWriter extends SSTable
{
@@ -115,7 +114,7 @@
/**
* Renames temporary SSTable files to valid data, index, and bloom filter files
*/
- public SSTableReader closeAndOpenReader(double cacheFraction) throws IOException
+ public SSTableReader closeAndOpenReader() throws IOException
{
// bloom filter
FileOutputStream fos = new FileOutputStream(filterFilename());
@@ -136,10 +135,7 @@
rename(filterFilename());
path = rename(path); // important to do this last since index & filter file names are derived from it
- ConcurrentLinkedHashMap<String,Long> keyCache = cacheFraction > 0
- ? SSTableReader.createKeyCache((int) (cacheFraction * keysWritten))
- : null;
- return new SSTableReader(path, partitioner, indexPositions, bf, keyCache);
+ return new SSTableReader(path, partitioner, indexPositions, bf);
}
static String rename(String tmpFilename)
@@ -154,7 +150,7 @@
SSTableWriter.rename(indexFilename(dataFileName));
SSTableWriter.rename(filterFilename(dataFileName));
dataFileName = SSTableWriter.rename(dataFileName);
- return SSTableReader.open(dataFileName, StorageService.getPartitioner(), DatabaseDescriptor.getKeysCachedFraction(parseTableName(dataFileName)));
+ return SSTableReader.open(dataFileName, StorageService.getPartitioner());
}
}
Modified: incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java?rev=810540&r1=810539&r2=810540&view=diff
==============================================================================
--- incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java (original)
+++ incubator/cassandra/trunk/test/unit/org/apache/cassandra/io/SSTableTest.java Wed Sep 2 15:04:16 2009
@@ -41,7 +41,7 @@
String key = Integer.toString(1);
writer.append(key, bytes);
- SSTableReader ssTable = writer.closeAndOpenReader(0.01);
+ SSTableReader ssTable = writer.closeAndOpenReader();
// verify
verifySingle(ssTable, bytes, key);
@@ -81,7 +81,7 @@
{
writer.append(key, map.get(key));
}
- SSTableReader ssTable = writer.closeAndOpenReader(0.01);
+ SSTableReader ssTable = writer.closeAndOpenReader();
// verify
verifyMany(ssTable, map);