You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@activemq.apache.org by eladko <el...@skyfence.com> on 2017/10/01 10:55:55 UTC
ActiveMQ OOM
*Hi all,
I'm getting OOM error in my broker:*
/INFO | jvm 1 | 2017/09/30 13:39:42 | INFO | Queue.34324.LIST Inactive
for longer than 60000 ms - removing ...
INFO | jvm 1 | 2017/09/30 15:34:04 | WARN | Store COMMIT FAILED:
INFO | jvm 1 | 2017/09/30 15:34:04 | java.lang.OutOfMemoryError: GC
overhead limit exceeded
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.disk.util.DataByteArrayOutputStream.<init>(DataByteArrayOutputStream.java:47)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.disk.page.Transaction$1.<init>(Transaction.java:279)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.disk.page.Transaction.openOutputStream(Transaction.java:279)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.disk.page.Transaction.store(Transaction.java:256)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.disk.index.BTreeIndex.storeNode(BTreeIndex.java:293)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.disk.index.BTreeNode.put(BTreeNode.java:394)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.disk.index.BTreeNode.put(BTreeNode.java:377)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.disk.index.BTreeIndex.put(BTreeIndex.java:189)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex.put(MessageDatabase.java:3692)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.MessageDatabase.updateIndex(MessageDatabase.java:1462)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.MessageDatabase$AddOperation.execute(MessageDatabase.java:3106)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.MessageDatabase$16.execute(MessageDatabase.java:1391)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.disk.page.Transaction.execute(Transaction.java:779)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:1387)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.MessageDatabase$10.visit(MessageDatabase.java:1226)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.data.KahaCommitCommand.visit(KahaCommitCommand.java:130)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:1208)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:1123)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:1099)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.store.kahadb.KahaDBTransactionStore.commit(KahaDBTransactionStore.java:301)[activemq-kahadb-store-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.transaction.LocalTransaction.commit(LocalTransaction.java:70)[activemq-broker-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.broker.TransactionBroker.commitTransaction(TransactionBroker.java:252)[activemq-broker-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.broker.BrokerFilter.commitTransaction(BrokerFilter.java:114)[activemq-broker-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.broker.BrokerFilter.commitTransaction(BrokerFilter.java:114)[activemq-broker-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.broker.TransportConnection.processCommitTransactionOnePhase(TransportConnection.java:529)[activemq-broker-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.command.TransactionInfo.visit(TransactionInfo.java:100)[activemq-client-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:330)[activemq-broker-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:194)[activemq-broker-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50)[activemq-client-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:125)[activemq-client-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:301)[activemq-client-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:04 | at
org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)[activemq-client-5.15.0.jar:5.15.0]
INFO | jvm 1 | 2017/09/30 15:34:13 | Exception in thread "ActiveMQ
Connection Dispatcher: tcp://11.1.1.77:43506" java.lang.OutOfMemoryError: GC
overhead limit exceeded
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.openwire.BooleanStream.<init>(BooleanStream.java:26)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.openwire.OpenWireFormat.marshal(OpenWireFormat.java:225)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.transport.tcp.TcpTransport.oneway(TcpTransport.java:193)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.transport.AbstractInactivityMonitor.doOnewaySend(AbstractInactivityMonitor.java:335)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.transport.AbstractInactivityMonitor.oneway(AbstractInactivityMonitor.java:317)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.transport.TransportFilter.oneway(TransportFilter.java:94)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.transport.WireFormatNegotiator.oneway(WireFormatNegotiator.java:116)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.broker.TransportConnection.dispatch(TransportConnection.java:1486)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.broker.TransportConnection.processDispatch(TransportConnection.java:971)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.broker.TransportConnection.iterate(TransportConnection.java:1021)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.thread.DedicatedTaskRunner.runTask(DedicatedTaskRunner.java:112)
INFO | jvm 1 | 2017/09/30 15:34:13 | at
org.apache.activemq.thread.DedicatedTaskRunner$1.run(DedicatedTaskRunner.java:42)
INFO | jvm 1 | 2017/09/30 15:34:19 | WARN | Async error occurred:
java.lang.OutOfMemoryError: GC overhead limit exceeded
INFO | jvm 1 | 2017/09/30 15:34:28 | WARN | Async error occurred:
java.lang.OutOfMemoryError: GC overhead limit exceeded
INFO | jvm 1 | 2017/09/30 15:39:42 | INFO | Queue.13757.LIST Inactive
for longer than 60000 ms - removing ...
I'm using persisted queues, and I can understand why the heap is getting so
big.
<persistenceAdapter>
<kahaDB directory="${activemq.data}/kahadb"
enableJournalDiskSyncs="false"/>
</persistenceAdapter>
Both Xmx and Xms are 2048MB, and my config in activemq.xml is:
<systemUsage>
<systemUsage>
<memoryUsage>
<memoryUsage percentOfJvmHeap="70" />
</memoryUsage>
<storeUsage>
<storeUsage limit="246 gb"/>
</storeUsage>
<tempUsage>
<tempUsage limit="50 gb"/>
</tempUsage>
</systemUsage>
</systemUsage>/
*Can someone explain what's being stored in the memory that can cause this
OOM?
Thanks all.*
--
Sent from: http://activemq.2283324.n4.nabble.com/ActiveMQ-User-f2341805.html