You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@activemq.apache.org by ch...@apache.org on 2011/11/22 18:46:17 UTC
svn commit: r1205104 - in /activemq/activemq-apollo/trunk:
apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/
apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/
apollo-distro/ apollo-distro/src/main/descriptors/ ...
Author: chirino
Date: Tue Nov 22 17:46:15 2011
New Revision: 1205104
URL: http://svn.apache.org/viewvc?rev=1205104&view=rev
Log:
Package the leveldb store and use it by default.
Removed:
activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/HALevelDBStoreDTO.java
Modified:
activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java
activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala
activemq/activemq-apollo/trunk/apollo-distro/pom.xml
activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml
activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml
activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index
activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java
activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml
activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md
Modified: activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java (original)
+++ activemq/activemq-apollo/trunk/apollo-bdb/src/main/scala/org/apache/activemq/apollo/broker/store/bdb/dto/package-info.java Tue Nov 22 17:46:15 2011
@@ -17,7 +17,6 @@
/**
* The JAXB POJOs for the
- * The JAXB POJOs for the
* <a href="http://activemq.apache.org/schema/activemq/apollo/xml-configuration.html">XML Configuration</a>
* of the ActiveMQ Broker.
*/
Modified: activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala (original)
+++ activemq/activemq-apollo/trunk/apollo-cli/src/main/scala/org/apache/activemq/apollo/cli/commands/Create.scala Tue Nov 22 17:46:15 2011
@@ -251,7 +251,11 @@ class Create extends Action {
}
def store_config = {
- if( can_load("com.sleepycat.je.Environment") ) {
+ if( can_load("org.fusesource.leveldbjni.JniDBFactory") ) {
+ """<!-- You can delete this element if you want to disable persistence for this virtual host -->
+ <leveldb_store directory="${apollo.base}/data"/>
+ """
+ } else if( can_load("com.sleepycat.je.Environment") ) {
"""<!-- You can delete this element if you want to disable persistence for this virtual host -->
<bdb_store directory="${apollo.base}/data"/>
"""
Modified: activemq/activemq-apollo/trunk/apollo-distro/pom.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-distro/pom.xml?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-distro/pom.xml (original)
+++ activemq/activemq-apollo/trunk/apollo-distro/pom.xml Tue Nov 22 17:46:15 2011
@@ -42,7 +42,7 @@
</dependency>
<dependency>
<groupId>org.apache.activemq</groupId>
- <artifactId>apollo-bdb</artifactId>
+ <artifactId>apollo-leveldb</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
@@ -52,6 +52,11 @@
</dependency>
<dependency>
<groupId>org.apache.activemq</groupId>
+ <artifactId>apollo-bdb</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.activemq</groupId>
<artifactId>apollo-tcp</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
Modified: activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml (original)
+++ activemq/activemq-apollo/trunk/apollo-distro/src/main/descriptors/common-bin.xml Tue Nov 22 17:46:15 2011
@@ -29,10 +29,9 @@
<include>org.scala-lang:scala-library</include>
<include>org.fusesource.hawtbuf:*</include>
<include>org.fusesource.hawtdispatch:*</include>
- <include>org.fusesource.hawtdb:*</include>
- <include>org.fusesource.jdbm:*</include>
<include>org.codehaus.jackson:jackson-mapper-asl</include>
<include>org.codehaus.jackson:jackson-core-asl</include>
+ <include>org.fusesource.leveldbjni:leveldbjni-all</include>
<!-- security stuff -->
<include>org.jasypt:jasypt</include>
Modified: activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml Tue Nov 22 17:46:15 2011
@@ -116,26 +116,6 @@
<build>
<plugins>
- <!-- include all the dependencies into the jar so it can run standalone -->
- <plugin>
- <groupId>org.fusesource.mvnplugins</groupId>
- <artifactId>maven-uberize-plugin</artifactId>
- <version>1.15</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals><goal>uberize</goal></goals>
- <configuration>
- <artifactSet>
- <includes>
- <include>org.fusesource.leveldbjni:leveldbjni-all</include>
- </includes>
- </artifactSet>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
<!-- <plugin>
<groupId>org.fusesource.mvnplugins</groupId>
<artifactId>maven-fab-plugin</artifactId>
Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/jaxb.index Tue Nov 22 17:46:15 2011
@@ -15,4 +15,4 @@
## limitations under the License.
## ---------------------------------------------------------------------------
LevelDBStoreDTO
-LevelDBStoreStatusDTO
+LevelDBStoreStatusDTO
\ No newline at end of file
Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/dto/package-info.java Tue Nov 22 17:46:15 2011
@@ -21,7 +21,7 @@
* of the ActiveMQ Broker.
*/
@javax.xml.bind.annotation.XmlSchema(
- namespace = "http://fabric.fusesource.org/apollo",
+ namespace = "http://activemq.apache.org/schema/activemq/apollo",
elementFormDefault = javax.xml.bind.annotation.XmlNsForm.QUALIFIED)
package org.apache.activemq.apollo.broker.store.leveldb.dto;
Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala Tue Nov 22 17:46:15 2011
@@ -33,7 +33,6 @@ import org.fusesource.hawtdispatch._
import org.apache.activemq.apollo.util.{TreeMap=>ApolloTreeMap}
import collection.immutable.TreeMap
import org.iq80.leveldb._
-import org.fusesource.leveldbjni.JniDBFactory._
import org.fusesource.leveldbjni.internal.Util
import org.fusesource.hawtbuf.{Buffer, AbstractVarIntSupport}
import java.util.concurrent.atomic.{AtomicReference, AtomicLong}
@@ -54,7 +53,10 @@ object LevelDBClient extends Log {
final val queue_prefix_array = Array(queue_prefix)
final val map_prefix_array = Array(map_prefix)
final val queue_entry_prefix_array = Array(queue_entry_prefix)
+
final val dirty_index_key = bytes(":dirty")
+ final val TRUE = bytes("true")
+ final val FALSE = bytes("false")
final val LOG_ADD_QUEUE = 1.toByte
final val LOG_REMOVE_QUEUE = 2.toByte
@@ -67,6 +69,8 @@ object LevelDBClient extends Log {
final val LOG_SUFFIX = ".log"
final val INDEX_SUFFIX = ".index"
+ def bytes(value:String) = value.getBytes("UTF-8")
+
import FileSupport._
def create_sequence_file(directory:File, id:Long, suffix:String) = directory / ("%016x%s".format(id, suffix))
@@ -139,6 +143,8 @@ class LevelDBClient(store: LevelDBStore)
var last_gc_duration = 0L
var in_gc = false
var gc_detected_log_usage = Map[Long, UsageCounter]()
+ var factory:DBFactory = org.fusesource.leveldbjni.JniDBFactory.factory
+
def dirty_index_file = directory / ("dirty"+INDEX_SUFFIX)
def temp_index_file = directory / ("temp"+INDEX_SUFFIX)
@@ -224,7 +230,7 @@ class LevelDBClient(store: LevelDBStore)
index = new RichDB(factory.open(dirty_index_file, index_options));
try {
- index.put(dirty_index_key, bytes("true"))
+ index.put(dirty_index_key, TRUE)
// Update the index /w what was stored on the logs..
var pos = last_index_snapshot_pos;
@@ -318,7 +324,7 @@ class LevelDBClient(store: LevelDBStore)
snapshot_rw_lock.writeLock().lock()
// Close the index so that it's files are not changed async on us.
- index.put(dirty_index_key, bytes("false"), new WriteOptions().sync(true))
+ index.put(dirty_index_key, FALSE, new WriteOptions().sync(true))
index.close
}
@@ -330,7 +336,7 @@ class LevelDBClient(store: LevelDBStore)
// re=open it..
retry {
index = new RichDB(factory.open(dirty_index_file, index_options));
- index.put(dirty_index_key, bytes("true"))
+ index.put(dirty_index_key, TRUE)
}
snapshot_rw_lock.writeLock().unlock()
}
Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/test/resources/org/apache/activemq/apollo/broker/store/leveldb/dto/simple.xml Tue Nov 22 17:46:15 2011
@@ -15,10 +15,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<broker xmlns="http://activemq.apache.org/schema/activemq/apollo"
- xmlns:fabric="http://fabric.fusesource.org/apollo">
+<broker xmlns="http://activemq.apache.org/schema/activemq/apollo">
<virtual_host enabled="true" id="vh-local">
<host_name>localhost</host_name>
- <fabric:leveldb_store directory="activemq-data"/>
+ <leveldb_store directory="activemq-data"/>
</virtual_host>
</broker>
Modified: activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md?rev=1205104&r1=1205103&r2=1205104&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md (original)
+++ activemq/activemq-apollo/trunk/apollo-website/src/documentation/user-manual.md Tue Nov 22 17:46:15 2011
@@ -342,12 +342,14 @@ cannot get swapped to disk.
${project_name} supports multiple message store implementations. The
implementations currently supported are:
+* [LevelDB Store](#LevelDB_Store) : is a file based message store implemented using the
+ [Google's LevelDB](http://en.wikipedia.org/wiki/LevelDB) library to maintain indexes into
+ log files holding the messages.
* [BDB Store](#BDB_Store) : is a file based message store implemented using the
[Sleepycat BDB](http://en.wikipedia.org/wiki/Berkeley_DB) library.
This is the most stable implementation.
* [JDBM2 Store](#JDBM2_Store) : is a file based message store implemented using the
[JDBM2](http://code.google.com/p/jdbm2/) library.
- This is the most stable implementation.
<!--
* [HawtDB Store](#HawtDB_Store) : is a file based message store implemented using the
@@ -356,6 +358,61 @@ implementations currently supported are:
debugger.
-->
+
+###### LevelDB Store
+
+The LevelDB store is the default store which a newly created Broker instance
+will use.
+
+It is enabled when your `virtual_host` element contains a `leveldb_store` element.
+
+{pygmentize:: xml}
+ ...
+ <virtual_host id="default">
+ ...
+ <leveldb_store directory="${apollo.base}/data"/>
+ ..
+ </virtual_host>
+ ...
+{pygmentize}
+
+A `leveldb_store` element may be configured with the following attributes:
+
+* `directory` : The directory which the store will use to hold it's data
+ files. The store will create the directory if it does not already
+ exist.
+* `flush_delay` : The flush delay is the amount of time in milliseconds
+ that a store will delay persisting a messaging unit of work in hopes
+ that it will be invalidated shortly thereafter by another unit of work
+ which would negate the operation.
+* `gc_interval` : How often to check to find log files which can be discarded
+ in seconds. The value defaults to 1800 (30 minutes).
+* `read_threads` : The number of concurrent IO reads to allow. The value
+ defaults to 10.
+* `sync` : If set to `false`, then the store does not sync logging operations to
+ disk. The value defaults to `true`.
+* `log_size` : The max size (in bytes) of each data log file before log file rotation
+ occurs. The value defaults to 104857600 (100 MB).
+* `log_write_buffer_size`: That maximum amount of log data to build up before writing
+ to the file system. The value defaults to 4194304 (4 MB).
+* `verify_checksums` : If set to `true` to force checksum verification of all
+ data that is read from the file system on behalf of a particular read. By
+ default, no such verification is done.
+* `paranoid_checks` : Make the store error out as soon as possible if it detects
+ internal corruption. The value defaults to false.
+* `index_max_open_files` : Number of open files that can be used by the index. The
+ value defaults to 1000.
+* `index_block_restart_interval` : Number keys between restart points for delta
+ encoding of keys. The value defaults to 16.
+* `index_write_buffer_size` : Amount of index data to build up in memory before
+ converting to a sorted on-disk file. The value defaults to 4194304 (4 MB).
+* `index_block_size` : The size of index data packed per block. The value defaults
+ to 4096 (4 K).
+* `index_cache_size` : The maximum amount of memory to use to cache index blocks.
+ The value defaults to 268435456 (256 MB).
+* `index_compression` : The type of compression to apply to the index blocks.
+ Can be `snappy` or `none`. The value defaults to `snappy`.
+
###### BDB Store
Apache cannot redistribute the BDB library due to the terms of it's