You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@activemq.apache.org by ch...@apache.org on 2013/02/21 15:36:22 UTC
svn commit: r1448654 - in /activemq/activemq-apollo/trunk:
apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/
apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/
apollo-website/src/documentation/
Author: chirino
Date: Thu Feb 21 14:36:22 2013
New Revision: 1448654
URL: http://svn.apache.org/r1448654
Log:
Fixes APLO-296: Support configuring the LevelDB auto compaction frequency
Modified:
activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala
activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java
activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md
Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala?rev=1448654&r1=1448653&r2=1448654&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/LevelDBClient.scala Thu Feb 21 14:36:22 2013
@@ -71,6 +71,7 @@ object LevelDBClient extends Log {
final val LOG_SUFFIX = ".log"
final val INDEX_SUFFIX = ".index"
+ var auto_compaction_ratio = 100
def bytes(value: String) = value.getBytes("UTF-8")
@@ -251,6 +252,7 @@ class LevelDBClient(store: LevelDBStore)
index_options.createIfMissing(true);
val paranoid_checks = config.paranoid_checks.getOrElse(false)
+ auto_compaction_ratio = OptionSupport(config.auto_compaction_ratio).getOrElse(100)
config.index_max_open_files.foreach(index_options.maxOpenFiles(_))
config.index_block_restart_interval.foreach(index_options.blockRestartInterval(_))
index_options.paranoidChecks(paranoid_checks)
@@ -1151,9 +1153,13 @@ class LevelDBClient(store: LevelDBStore)
}
}
+ // APLO-245: lets try to detect when leveldb needs a compaction..
+ def detect_if_compact_needed:Unit = {
- def detect_if_compact_needed = {
- // APLO-245: lets try to detect when leveldb needs a compaction..
+ // auto compaction might be disabled...
+ if ( auto_compaction_ratio <= 0 ) {
+ return
+ }
// How much space is the dirty index using??
var index_usage = 0L
@@ -1174,7 +1180,7 @@ class LevelDBClient(store: LevelDBStore)
// After running some load we empirically found that a healthy ratio is between 12 and 25 bytes per entry.
// lets compact if we go way over the healthy ratio.
- if( ratio > 100 ) {
+ if( ratio > auto_compaction_ratio ) {
index.compact_needed = true
}
} else if( index_usage > 1024*1024*5 ) {
@@ -1195,9 +1201,11 @@ class LevelDBClient(store: LevelDBStore)
// Lets compact the leveldb index if it looks like we need to.
if( index.compact_needed ) {
- info("Compacting the leveldb index at: "+dirty_index_file)
+ debug("Compacting the leveldb index at: %s", dirty_index_file)
+ val start = System.nanoTime()
index.compact
- info("Compaction completed")
+ val duration = System.nanoTime() - start;
+ info("Compacted the leveldb index at: %s in %.2f ms", dirty_index_file, (duration / 1000000.0))
}
val empty_journals = log.log_infos.keySet.toSet -- log_refs.keySet
Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java?rev=1448654&r1=1448653&r2=1448654&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/LevelDBStoreDTO.java Thu Feb 21 14:36:22 2013
@@ -78,6 +78,9 @@ public class LevelDBStoreDTO extends Sto
@XmlAttribute(name="index_factory")
public String index_factory;
+ @XmlAttribute(name="auto_compaction_ratio")
+ public Integer auto_compaction_ratio;
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
Modified: activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md?rev=1448654&r1=1448653&r2=1448654&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md (original)
+++ activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md Thu Feb 21 14:36:22 2013
@@ -639,6 +639,11 @@ A `leveldb_store` element may be configu
Can be `snappy` or `none`. The value defaults to `snappy`.
* `log_compression` : The type of compression to apply to the log records.
Can be `snappy` or `none`. The value defaults to `snappy`.
+* `auto_compaction_ratio`: This ratio is used to determine when to compact
+ the leveldb indexes. When you take the ratio of disk space used by the leveldb
+ indexes to the number queue entries and it exceeds the configured
+ `auto_compaction_ratio` then the leveldb index will be scheduled for compaction.
+ If set to 0, then auto compactions are disabled. The value defaults to 100.
### Support Platforms