svn commit: r1439549 - /activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala

[ch...@apache.org]

Author: chirino
Date: Mon Jan 28 18:12:31 2013
New Revision: 1439549

URL: http://svn.apache.org/viewvc?rev=1439549&view=rev
Log:
APLO-245: The LevelDB store does not seem to get cleaned/compacted

We now periodically monitor the size of the index and compact if it has grown too large relative to the number of entries we have stored in it.

Modified:
    activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala?rev=1439549&r1=1439548&r2=1439549&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala Mon Jan 28 18:12:31 2013
@@ -1134,6 +1134,39 @@ class LevelDBClient(store: LevelDBStore)
     }
   }
 
+
+  def detect_if_compact_needed = {
+    // APLO-245: lets try to detect when leveldb needs a compaction..
+
+    // How much space is the dirty index using??
+    var index_usage = 0L
+    for( file <- dirty_index_file.recursive_list ) {
+      if(!file.isDirectory) {
+        index_usage += file.length()
+      }
+    }
+    // Lets use the log_refs to get a rough estimate on how many entries are store in leveldb.
+    var index_queue_entries=0L
+    for ( (_, count) <- log_refs ) {
+      index_queue_entries += count.get()
+    }
+
+    if ( index_queue_entries > 0 ) {
+      val ratio = (index_usage*1.0f/index_queue_entries)
+      // println("usage: index_usage:%d, index_queue_entries:%d, ratio: %f".format(index_usage, index_queue_entries, ratio))
+
+      // After running some load we empirically found that a healthy ratio is between 12 and 25 bytes per entry.
+      // lets compact if we go way over the healthy ratio.
+      if( ratio > 50 ) {
+        index.compact_needed = true
+      }
+    } else if( index_usage > 1024*1024*5 )  {
+      // at most the index should have 1 full level file.
+      index.compact_needed = true
+    }
+
+  }
+
   def gc: Unit = {
 
     // TODO:
@@ -1141,6 +1174,8 @@ class LevelDBClient(store: LevelDBStore)
     //
     import collection.JavaConversions._
 
+    detect_if_compact_needed
+
     // Lets compact the leveldb index if it looks like we need to.
     if( index.compact_needed ) {
       index.compact