You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@activemq.apache.org by lhf刘海峰 <hf...@Ctrip.com> on 2013/01/05 11:15:09 UTC
Why it's losing data? more than 10%! (activemq 5.6)
Hi,
I am using activemq 5.6.0 to collect some event data and dispatch them to the analyzer cluster. I am using non-persist queue, store cursor and kahadb. Now I found that it’s losing data heavily, more than 10%. There is about 100+ million messages per day, and messages are consumed immediately. The CPU and memory usage both seems OK. Network load is not heavy I think.
I have no idea to locate the problem. Please help! The following are the JVM Options and ActiveMQ configuration file:
JVM Opts:
-Xms8G
-Xmx8G
-XX:NewRatio=1
-XX:PermSize=64M
-XX:MaxPermSize=64M
-XX:+UseParallelGC
-XX:+UseParallelOldGC
-XX:ParallelGCThreads=10
-XX:InitialTenuringThreshold=7
-XX:MaxTenuringThreshold=15
Configuration:
<beans
xmlns="http://www.springframework.org/schema/beans"
xmlns:amq="http://activemq.apache.org/schema/core"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd">
<bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="locations">
<value>file:${activemq.conf}/credentials.properties</value>
</property>
</bean>
<broker xmlns="http://activemq.apache.org/schema/core"
useJmx="true"
brokerName="activemq"
dataDirectory="${activemq.data}"
advisorySupport="true"
persistent="false">
<destinationPolicy>
<policyMap>
<policyEntries>
<policyEntry topic=">"
producerFlowControl="false"
maxPageSize="500"
topicPrefetch="1000"
durableTopicPrefetch="1000"
memoryLimit="2mb">
<dispatchPolicy>
<roundRobinDispatchPolicy />
</dispatchPolicy>
<pendingSubscriberPolicy>
<vmCursor />
</pendingSubscriberPolicy>
<pendingDurableSubscriberPolicy>
<storeDurableSubscriberCursor />
</pendingDurableSubscriberPolicy>
</policyEntry>
<policyEntry queue=">"
producerFlowControl="false"
maxPageSize="500"
queuePrefetch="1000"
memoryLimit="2mb">
<dispatchPolicy>
<simpleDispatchPolicy />
</dispatchPolicy>
<pendingQueuePolicy>
<storeCursor />
</pendingQueuePolicy>
</policyEntry>
</policyEntries>
</policyMap>
</destinationPolicy>
<managementContext>
<managementContext createConnector="false"/>
</managementContext>
<persistenceAdapter>
<kahaDB directory="${activemq.data}/kahadb"
indexCacheSize="10000"
indexWriteBatchSize="1000"
journalMaxFileLength="128mb"
enableJournalDiskSyncs="true"
enableIndexWriteAsync="true" />
</persistenceAdapter>
<systemUsage>
<systemUsage>
<memoryUsage>
<memoryUsage limit="512 mb"/>
</memoryUsage>
<storeUsage>
<storeUsage limit="2048 gb"/>
</storeUsage>
<tempUsage>
<tempUsage limit="10 gb"/>
</tempUsage>
</systemUsage>
</systemUsage>
<transportConnectors>
<transportConnector name="openwire" uri="nio://0.0.0.0:61616" />
</transportConnectors>
</broker>
<import resource="jetty.xml"/>
</beans>
Re: Why it's losing data? more than 10%! (activemq 5.6)
Posted by Gary Tully <ga...@gmail.com>.
how do you detect loss? Please provide some more context.
On 5 January 2013 10:15, lhf刘海峰 <hf...@ctrip.com> wrote:
> Hi,
> I am using activemq 5.6.0 to collect some event data and dispatch them to
> the analyzer cluster. I am using non-persist queue, store cursor and
> kahadb. Now I found that it’s losing data heavily, more than 10%. There is
> about 100+ million messages per day, and messages are consumed immediately.
> The CPU and memory usage both seems OK. Network load is not heavy I think.
>
> I have no idea to locate the problem. Please help! The following are the
> JVM Options and ActiveMQ configuration file:
>
> JVM Opts:
>
> -Xms8G
> -Xmx8G
> -XX:NewRatio=1
> -XX:PermSize=64M
> -XX:MaxPermSize=64M
> -XX:+UseParallelGC
> -XX:+UseParallelOldGC
> -XX:ParallelGCThreads=10
> -XX:InitialTenuringThreshold=7
> -XX:MaxTenuringThreshold=15
>
> Configuration:
>
> <beans
> xmlns="http://www.springframework.org/schema/beans"
> xmlns:amq="http://activemq.apache.org/schema/core"
> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
> xsi:schemaLocation="http://www.springframework.org/schema/beans
> http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
> http://activemq.apache.org/schema/core
> http://activemq.apache.org/schema/core/activemq-core.xsd">
> <bean
> class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
> <property name="locations">
> <value>file:${activemq.conf}/credentials.properties</value>
> </property>
> </bean>
> <broker xmlns="http://activemq.apache.org/schema/core"
> useJmx="true"
> brokerName="activemq"
> dataDirectory="${activemq.data}"
> advisorySupport="true"
> persistent="false">
> <destinationPolicy>
> <policyMap>
> <policyEntries>
> <policyEntry topic=">"
> producerFlowControl="false"
> maxPageSize="500"
> topicPrefetch="1000"
> durableTopicPrefetch="1000"
> memoryLimit="2mb">
> <dispatchPolicy>
> <roundRobinDispatchPolicy />
> </dispatchPolicy>
> <pendingSubscriberPolicy>
> <vmCursor />
> </pendingSubscriberPolicy>
> <pendingDurableSubscriberPolicy>
> <storeDurableSubscriberCursor />
> </pendingDurableSubscriberPolicy>
> </policyEntry>
> <policyEntry queue=">"
> producerFlowControl="false"
> maxPageSize="500"
> queuePrefetch="1000"
> memoryLimit="2mb">
> <dispatchPolicy>
> <simpleDispatchPolicy />
> </dispatchPolicy>
> <pendingQueuePolicy>
> <storeCursor />
> </pendingQueuePolicy>
> </policyEntry>
> </policyEntries>
> </policyMap>
> </destinationPolicy>
> <managementContext>
> <managementContext createConnector="false"/>
> </managementContext>
> <persistenceAdapter>
> <kahaDB directory="${activemq.data}/kahadb"
> indexCacheSize="10000"
> indexWriteBatchSize="1000"
> journalMaxFileLength="128mb"
> enableJournalDiskSyncs="true"
> enableIndexWriteAsync="true" />
> </persistenceAdapter>
> <systemUsage>
> <systemUsage>
> <memoryUsage>
> <memoryUsage limit="512 mb"/>
> </memoryUsage>
> <storeUsage>
> <storeUsage limit="2048 gb"/>
> </storeUsage>
> <tempUsage>
> <tempUsage limit="10 gb"/>
> </tempUsage>
> </systemUsage>
> </systemUsage>
> <transportConnectors>
> <transportConnector name="openwire" uri="nio://0.0.0.0:61616" />
> </transportConnectors>
> </broker>
> <import resource="jetty.xml"/>
> </beans>
>
>
--
http://redhat.com
http://blog.garytully.com