You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/10/02 20:40:27 UTC
svn commit: r821127 - in /incubator/cassandra/trunk: conf/
src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/
src/java/org/apache/cassandra/service/
Author: jbellis
Date: Fri Oct 2 18:40:27 2009
New Revision: 821127
URL: http://svn.apache.org/viewvc?rev=821127&view=rev
Log:
add MemtableFlushAfterMinutes, a global replacement for the old per-CF FlushPeriodInMinutes setting
patch by jbellis; reviewed by junrao for CASSANDRA-463
Modified:
incubator/cassandra/trunk/conf/storage-conf.xml
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java
Modified: incubator/cassandra/trunk/conf/storage-conf.xml
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/conf/storage-conf.xml?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/conf/storage-conf.xml (original)
+++ incubator/cassandra/trunk/conf/storage-conf.xml Fri Oct 2 18:40:27 2009
@@ -58,15 +58,8 @@
~
~ (To get the closest approximation to 0.3-style supercolumns, you
~ would use CompareWith=UTF8Type CompareSubcolumnsWith=LongType.)
-
- ~ If FlushPeriodInMinutes is positive, it will be
- ~ flushed to disk with that period even if the memtable has not
- ~ exceeded the size or count thresholds, so that it does not
- ~ prevent commitlog segments from being purged.
-->
- <ColumnFamily CompareWith="BytesType"
- Name="Standard1"
- FlushPeriodInMinutes="60"/>
+ <ColumnFamily CompareWith="BytesType" Name="Standard1"/>
<ColumnFamily CompareWith="UTF8Type" Name="Standard2"/>
<ColumnFamily CompareWith="TimeUUIDType" Name="StandardByUUID1"/>
<ColumnFamily ColumnType="Super"
@@ -243,6 +236,16 @@
~ setting. Use with MemtableSizeInMB to tune memory usage.
-->
<MemtableObjectCountInMillions>0.1</MemtableObjectCountInMillions>
+ <!--
+ ~ The maximum time to leave a dirty memtable unflushed.
+ ~ (While any affected columnfamilies have unflushed data from a
+ ~ commit log segment, that segment cannot be deleted.)
+ ~ This needs to be large enough that it won't cause a flush storm
+ ~ of all your memtables flushing at once because none has hit
+ ~ the size or count thresholds yet. For production, a larger
+ ~ value such as 1440 is recommended.
+ -->
+ <MemtableFlushAfterMinutes>60</MemtableFlushAfterMinutes>
<!--
~ Unlike most systems, in Cassandra writes are faster than reads, so
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Fri Oct 2 18:40:27 2009
@@ -36,7 +36,6 @@
public String n_columnKey;
public String n_columnValue;
public String n_columnTimestamp;
- public int flushPeriodInMinutes = 0; // flush interval, if <=0, no periodic flusher is scheduled
// a quick and dirty pretty printer for describing the column family...
public String pretty()
@@ -51,7 +50,6 @@
desc += "Column Family Type: " + columnType + "\n" +
"Columns Sorted By: " + comparator + "\n";
- desc += "flush period: " + flushPeriodInMinutes + " minutes\n";
return desc;
}
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Fri Oct 2 18:40:27 2009
@@ -20,7 +20,6 @@
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.dht.IPartitioner;
@@ -101,8 +100,8 @@
/* if the size of columns or super-columns are more than this, indexing will kick in */
private static int columnIndexSizeInKB_;
- /* Number of hours to keep a memtable in memory */
- private static int memtableLifetime_ = 6;
+ /* Number of minutes to keep a memtable in memory */
+ private static int memtableLifetimeMs_ = 60 * 60 * 1000;
/* Size of the memtable in memory before it is dumped */
private static int memtableSize_ = 128;
/* Number of objects in millions in the memtable before it is dumped */
@@ -320,9 +319,9 @@
/* Number of days to keep the memtable around w/o flushing */
- String lifetime = xmlUtils.getNodeValue("/Storage/MemtableLifetimeInDays");
- if ( lifetime != null )
- memtableLifetime_ = Integer.parseInt(lifetime);
+ String lifetime = xmlUtils.getNodeValue("/Storage/MemtableFlushAfterMinutes");
+ if (lifetime != null)
+ memtableLifetimeMs_ = Integer.parseInt(lifetime) * 60 * 1000;
/* Size of the memtable in memory in MB before it is dumped */
String memtableSize = xmlUtils.getNodeValue("/Storage/MemtableSizeInMB");
@@ -471,13 +470,6 @@
throw new ConfigurationException("CompareSubcolumnsWith is only a valid attribute on super columnfamilies (not regular columnfamily " + cfName + ")");
}
- // see if flush period is set
- String flushPeriodInMinutes = XMLUtils.getAttributeValue(columnFamily, "FlushPeriodInMinutes");
- int flushPeriod=0;
- if ( flushPeriodInMinutes != null )
- flushPeriod = Integer.parseInt(flushPeriodInMinutes);
-
-
// Parse out user-specified logical names for the various dimensions
// of a the column family from the config.
String n_superColumnMap = xmlUtils.getNodeValue(xqlCF + "SuperColumnMap");
@@ -525,8 +517,7 @@
cfMetaData.n_superColumnKey = n_superColumnKey;
cfMetaData.n_superColumnMap = n_superColumnMap;
}
- cfMetaData.flushPeriodInMinutes = flushPeriod;
-
+
tableToCFMetaDataMap_.get(tName).put(cfName, cfMetaData);
}
}
@@ -538,14 +529,12 @@
CFMetaData data = new CFMetaData();
data.columnType = "Standard";
data.comparator = new UTF8Type();
- data.flushPeriodInMinutes = 1;
systemMetadata.put(SystemTable.LOCATION_CF, data);
data = new CFMetaData();
data.columnType = "Super";
data.comparator = new UTF8Type();
data.subcolumnComparator = new BytesType();
- data.flushPeriodInMinutes = 10;
systemMetadata.put(HintedHandOffManager.HINTS_CF, data);
tableToCFMetaDataMap_.put(Table.SYSTEM_TABLE, systemMetadata);
@@ -673,9 +662,9 @@
return columnIndexSizeInKB_ * 1024;
}
- public static int getMemtableLifetime()
+ public static int getMemtableLifetimeMS()
{
- return memtableLifetime_;
+ return memtableLifetimeMs_;
}
public static String getInitialToken()
@@ -748,16 +737,6 @@
return cfMetaData.columnType;
}
- public static int getFlushPeriod(String tableName, String columnFamilyName)
- {
- assert tableName != null;
- CFMetaData cfMetaData = getCFMetaData(tableName, columnFamilyName);
-
- if (cfMetaData == null)
- return 0;
- return cfMetaData.flushPeriodInMinutes;
- }
-
public static List<String> getTables()
{
return tables_;
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Fri Oct 2 18:40:27 2009
@@ -85,7 +85,6 @@
new LinkedBlockingQueue<Runnable>(),
new NamedThreadFactory("FLUSH-WRITER-POOL"));
private static ExecutorService commitLogUpdater_ = new DebuggableThreadPoolExecutor("MEMTABLE-POST-FLUSHER");
- private static Timer flushTimer_ = new Timer("FLUSH-TIMER");
private final String table_;
public final String columnFamily_;
@@ -223,26 +222,6 @@
{
HintedHandOffManager.instance().scheduleHandoffsFor(this);
}
-
- // schedule periodic flusher if required
- int flushPeriodMS = DatabaseDescriptor.getFlushPeriod(table_, columnFamily_) * 60 * 1000;
- if (flushPeriodMS > 0)
- {
- flushTimer_.schedule(new TimerTask()
- {
- public void run()
- {
- try
- {
- forceFlush();
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
- }, flushPeriodMS, flushPeriodMS);
- }
}
/*
@@ -417,6 +396,12 @@
binaryMemtable_.get().put(key, buffer);
}
+ public void forceFlushIfExpired() throws IOException
+ {
+ if (memtable_.isExpired())
+ forceFlush();
+ }
+
public Future<?> forceFlush() throws IOException
{
if (memtable_.isClean())
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Fri Oct 2 18:40:27 2009
@@ -346,4 +346,9 @@
{
columnFamilies_.clear();
}
+
+ public boolean isExpired()
+ {
+ return System.currentTimeMillis() > creationTime_ + DatabaseDescriptor.getMemtableLifetimeMS();
+ }
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Fri Oct 2 18:40:27 2009
@@ -53,6 +53,8 @@
/* we use this lock to drain updaters before calling a flush. */
static final ReentrantReadWriteLock flusherLock_ = new ReentrantReadWriteLock(true);
+ private static Timer flushTimer_ = new Timer("FLUSH-TIMER");
+
/*
* This class represents the metadata of this Table. The metadata
* is basically the column family name and the ID associated with
@@ -513,6 +515,26 @@
{
columnFamilyStores_.put(columnFamily, ColumnFamilyStore.getColumnFamilyStore(table, columnFamily));
}
+
+ // check 10x as often as the lifetime, so we can exceed lifetime by 10% at most
+ int checkMs = DatabaseDescriptor.getMemtableLifetimeMS() / 10;
+ flushTimer_.schedule(new TimerTask()
+ {
+ public void run()
+ {
+ for (ColumnFamilyStore cfs : columnFamilyStores_.values())
+ {
+ try
+ {
+ cfs.forceFlushIfExpired();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }, checkMs, checkMs);
}
boolean isApplicationColumnFamily(String columnFamily)
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java Fri Oct 2 18:40:27 2009
@@ -572,7 +572,6 @@
{
columnMap.put("CompareSubcolumnsWith", columnFamilyMetaData.subcolumnComparator.getClass().getName());
}
- columnMap.put("FlushPeriodInMinutes", columnFamilyMetaData.flushPeriodInMinutes + "");
columnFamiliesMap.put(columnFamilyMetaData.cfName, columnMap);
}
return columnFamiliesMap;