You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@jackrabbit.apache.org by "Martinez, Antonio (Antonio)" <an...@alcatel-lucent.com> on 2009/12/07 22:45:38 UTC
Deadlock in Jackrabbit 1.4.4 (lucene)
Hello,
We have been running Jackrabbit 1.4.4 for some time with no problem.
But today I have an issue with many threads blocked.
A thread dump showed that there is a deadlock in lucene. I created ticket http://issues.apache.org/jira/browse/JCR-2426
Any help is appreciated. This has happened in a production setup, which we had to restart.
Thanks,
Antonio
-------------------------
Found one Java-level deadlock:
=============================
"RMI TCP Connection(106261)-10.196.27.37":
waiting to lock monitor 0x03764818 (object 0x80c376b0, a org.apache.lucene.store.FSDirectory$FSIndexInput),
which is held by "jmssecondaryApplnJobExecutor-7"
"jmssecondaryApplnJobExecutor-7":
waiting to lock monitor 0x050ad618 (object 0x808fa710, a org.apache.lucene.index.SegmentReader),
which is held by "IndexMerger"
"IndexMerger":
waiting to lock monitor 0x03764818 (object 0x80c376b0, a org.apache.lucene.store.FSDirectory$FSIndexInput),
which is held by "jmssecondaryApplnJobExecutor-7"
Java stack information for the threads listed above:
===================================================
"RMI TCP Connection(106261)-10.196.27.37":
at org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
- waiting to lock <0x80c376b0> (a org.apache.lucene.store.FSDirectory$FSIndexInput)
at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:143)
at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
at org.apache.lucene.store.IndexInput.readVInt(IndexInput.java:57)
at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:63)
at org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:122)
at org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:153)
at org.apache.lucene.index.TermInfosReader.scanEnum(TermInfosReader.java:164)
at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:158)
at org.apache.lucene.index.SegmentReader.docFreq(SegmentReader.java:352)
at org.apache.lucene.index.FilterIndexReader.docFreq(FilterIndexReader.java:176)
at org.apache.lucene.index.FilterIndexReader.docFreq(FilterIndexReader.java:176)
at org.apache.lucene.index.FilterIndexReader.docFreq(FilterIndexReader.java:176)
at org.apache.lucene.index.MultiReader.docFreq(MultiReader.java:220)
at org.apache.lucene.index.MultiReader.docFreq(MultiReader.java:220)
at org.apache.lucene.search.IndexSearcher.docFreq(IndexSearcher.java:87)
at org.apache.lucene.search.Similarity.idf(Similarity.java:459)
at org.apache.lucene.search.TermQuery$TermWeight.<init>(TermQuery.java:44)
at org.apache.lucene.search.TermQuery.createWeight(TermQuery.java:145)
at org.apache.lucene.search.BooleanQuery$BooleanWeight.<init>(BooleanQuery.java:187)
at org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:353)
at org.apache.lucene.search.BooleanQuery$BooleanWeight.<init>(BooleanQuery.java:187)
at org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:353)
at org.apache.lucene.search.Query.weight(Query.java:95)
at org.apache.lucene.search.Hits.<init>(Hits.java:50)
at org.apache.lucene.search.Searcher.search(Searcher.java:46)
at org.apache.lucene.search.Searcher.search(Searcher.java:38)
at org.apache.jackrabbit.core.query.lucene.SearchIndex.executeQuery(SearchIndex.java:691)
at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.executeQuery(QueryResultImpl.java:242)
at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.getResults(QueryResultImpl.java:290)
at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.<init>(QueryResultImpl.java:192)
at org.apache.jackrabbit.core.query.lucene.QueryImpl.execute(QueryImpl.java:138)
at org.apache.jackrabbit.core.query.QueryImpl.execute(QueryImpl.java:176)
at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl$2.execute(InventoryDiscoveryServiceImpl.java:422)
at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl.execute(InventoryDiscoveryServiceImpl.java:316)
at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl.queryData(InventoryDiscoveryServiceImpl.java:407)
at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl.queryData(InventoryDiscoveryServiceImpl.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:36)
at sun.reflect.GeneratedMethodAccessor107.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:243)
at javax.management.modelmbean.RequiredModelMBean.invokeMethod(RequiredModelMBean.java:1074)
at javax.management.modelmbean.RequiredModelMBean.invoke(RequiredModelMBean.java:955)
at org.springframework.jmx.export.SpringModelMBean.invoke(SpringModelMBean.java:88)
at org.jboss.mx.server.RawDynamicInvoker.invoke(RawDynamicInvoker.java:164)
at org.jboss.mx.modelmbean.RequiredModelMBeanInvoker.invoke(RequiredModelMBeanInvoker.java:127)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
at sun.reflect.GeneratedMethodAccessor6569.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.jmx.connector.invoker.InvokerAdaptorService.invoke(InvokerAdaptorService.java:266)
at sun.reflect.GeneratedMethodAccessor163.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
at org.jboss.mx.interceptor.AbstractInterceptor.invoke(AbstractInterceptor.java:133)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)
at org.jboss.mx.interceptor.ModelMBeanOperationInterceptor.invoke(ModelMBeanOperationInterceptor.java:142)
at com.alcatel.axs.applications.nelist.SnmpStatsInterceptor.invoke(SnmpStatsInterceptor.java:21)
at com.alcatel.axs.container.jmx.PayloadServerInterceptor.invoke(PayloadServerInterceptor.java:32)
at com.alcatel.axs.app.security.session.SessionIdPostAuthenticationServerInterceptor.invoke(SessionIdPostAuthenticationServerInterceptor.java:118)
at org.jboss.jmx.connector.invoker.AuthenticationInterceptor.invoke(AuthenticationInterceptor.java:108)
at com.alcatel.axs.app.security.session.AxsAuthenticationInterceptor.invoke(AxsAuthenticationInterceptor.java:80)
at com.alcatel.axs.app.security.session.SessionIdPreAuthenticationServerInterceptor.invoke(SessionIdPreAuthenticationServerInterceptor.java:86)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
at org.jboss.invocation.jrmp.server.JRMPInvokerHA.invoke(JRMPInvokerHA.java:177)
at sun.reflect.GeneratedMethodAccessor636.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:305)
at sun.rmi.transport.Transport$1.run(Transport.java:159)
at java.security.AccessController.doPrivileged(Native Method)
at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"jmssecondaryApplnJobExecutor-7":
at org.apache.lucene.index.SegmentReader.hasNorms(SegmentReader.java:421)
- waiting to lock <0x808fa710> (a org.apache.lucene.index.SegmentReader)
at org.apache.lucene.index.FilterIndexReader.hasNorms(FilterIndexReader.java:147)
at org.apache.lucene.index.FilterIndexReader.hasNorms(FilterIndexReader.java:147)
at org.apache.lucene.index.FilterIndexReader.hasNorms(FilterIndexReader.java:147)
at org.apache.lucene.index.MultiReader.hasNorms(MultiReader.java:161)
at org.apache.lucene.index.MultiReader.hasNorms(MultiReader.java:161)
at org.apache.lucene.index.MultiReader.norms(MultiReader.java:177)
- locked <0x80f07f60> (a org.apache.jackrabbit.core.query.lucene.SearchIndex$CombinedIndexReader)
at org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:69)
at org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:233)
at org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:233)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:143)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:113)
at org.apache.lucene.search.Hits.getMoreDocs(Hits.java:74)
at org.apache.lucene.search.Hits.<init>(Hits.java:53)
at org.apache.lucene.search.Searcher.search(Searcher.java:46)
at org.apache.lucene.search.Searcher.search(Searcher.java:38)
at org.apache.jackrabbit.core.query.lucene.SearchIndex.executeQuery(SearchIndex.java:691)
at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.executeQuery(QueryResultImpl.java:242)
at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.getResults(QueryResultImpl.java:290)
at org.apache.jackrabbit.core.query.lucene.QueryResultImpl.<init>(QueryResultImpl.java:192)
at org.apache.jackrabbit.core.query.lucene.QueryImpl.execute(QueryImpl.java:138)
at org.apache.jackrabbit.core.query.QueryImpl.execute(QueryImpl.java:176)
at com.alcatel.axs.app.idm.nelist.InventoryNeListServiceImpl.findNeNode(InventoryNeListServiceImpl.java:621)
at com.alcatel.axs.app.idm.nelist.InventoryNeListServiceImpl.collectionBegin(InventoryNeListServiceImpl.java:137)
at com.alcatel.axs.app.idm.InventoryDiscoveryServiceImpl.doDiscoveryNow(InventoryDiscoveryServiceImpl.java:219)
at sun.reflect.GeneratedMethodAccessor7068.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:36)
at sun.reflect.GeneratedMethodAccessor107.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:243)
at javax.management.modelmbean.RequiredModelMBean.invokeMethod(RequiredModelMBean.java:1074)
at javax.management.modelmbean.RequiredModelMBean.invoke(RequiredModelMBean.java:955)
at org.springframework.jmx.export.SpringModelMBean.invoke(SpringModelMBean.java:88)
at org.jboss.mx.server.RawDynamicInvoker.invoke(RawDynamicInvoker.java:164)
at org.jboss.mx.modelmbean.RequiredModelMBeanInvoker.invoke(RequiredModelMBeanInvoker.java:127)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
at org.jboss.system.server.jmx.LazyMBeanServer.invoke(LazyMBeanServer.java:291)
at javax.management.MBeanServerInvocationHandler.invoke(MBeanServerInvocationHandler.java:288)
at $Proxy638.doDiscoveryNow(Unknown Source)
at com.alcatel.axs.app.idm.InventoryJob.execute(InventoryJob.java:41)
at com.alcatel.axs.container.jobmanager.ManagedJob.run(ManagedJob.java:55)
at com.alcatel.axs.container.jobmanager.JMSJobExecutor.onMessage(JMSJobExecutor.java:50)
at org.springframework.jms.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:531)
at org.springframework.jms.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:466)
at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:435)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:322)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:260)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:944)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:868)
at java.lang.Thread.run(Thread.java:619)
"IndexMerger":
at org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
- waiting to lock <0x80c376b0> (a org.apache.lucene.store.FSDirectory$FSIndexInput)
at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:143)
at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
at org.apache.lucene.store.IndexInput.readInt(IndexInput.java:47)
at org.apache.lucene.store.IndexInput.readLong(IndexInput.java:70)
at org.apache.lucene.index.FieldsReader.doc(FieldsReader.java:104)
at org.apache.lucene.index.SegmentReader.document(SegmentReader.java:333)
- locked <0x808fa710> (a org.apache.lucene.index.SegmentReader)
at org.apache.lucene.index.FilterIndexReader.document(FilterIndexReader.java:130)
at org.apache.lucene.index.FilterIndexReader.document(FilterIndexReader.java:130)
at org.apache.lucene.index.FilterIndexReader.document(FilterIndexReader.java:130)
at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:207)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:97)
at org.apache.lucene.index.IndexWriter.addIndexes(IndexWriter.java:1632)
- locked <0x808f96d0> (a org.apache.lucene.index.IndexWriter)
at org.apache.jackrabbit.core.query.lucene.PersistentIndex.addIndexes(PersistentIndex.java:107)
at org.apache.jackrabbit.core.query.lucene.IndexMerger.run(IndexMerger.java:298)
Found 1 deadlock.