You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-user@lucene.apache.org by damu_verse <da...@gmail.com> on 2009/03/03 14:13:29 UTC
Tomcat Threads are BLOCKED after some time
Hi All,
We Have used Lucene as our Search Engine and all our applications
are deployed onto tomcat and running with thread pool size of 200.
Java Version - 1.6.0-rc
Lucene Version - 2.3.2
Tomcat Version - 6.0.14
OS - Red Hat Enterprise Linux ES release 4 (Nahant Update 5)
kernel - 2.6.9-55.0.2.ELsmp
RAM - 4 GB
Tomcat Memory - 1.5 GB
Index Size - 2 GB
After 10-12 hrs of tomcat running, tomcat
becomes irresponsive. After doing core dump of tomcat process We observed
that all tomcat threads are blocked (Thread-pool size-200). none of the
tomcat threads are in runnable state.
each thread at the time of the core dump is in BLOCKED state...The following
are the stack trace of blocked.
"MultiSearcher thread #3" daemon prio=10 tid=0x337ddc00 nid=0x4827 waiting
for monitor entry [0x2f2f0000..0x2f2f0ea0]
java.lang.Thread.State: BLOCKED (on object monitor)
at
org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
- waiting to lock <0x45d49d88> (a
org.apache.lucene.store.FSDirectory$FSIndexInput)
at
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
at
org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
at org.apache.lucene.store.IndexInput.readVInt(IndexInput.java:76)
at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:63)
at org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:123)
at org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:154)
at
org.apache.lucene.index.TermInfosReader.scanEnum(TermInfosReader.java:223)
at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:217)
at org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:54)
at org.apache.lucene.index.IndexReader.termDocs(IndexReader.java:668)
at
org.apache.lucene.search.ConstantScoreTermQuery$TermWeight.scorer(ConstantScoreTermQuery.java:63)
at
org.apache.lucene.search.VBooleanQuery$BooleanWeight.scorer(VBooleanQuery.java:276)
at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:232)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:143)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:124)
at
org.apache.lucene.search.MultiSearcherThread.run(ParallelMultiSearcher.java:250)
"http-8080-194" daemon prio=10 tid=0x08927800 nid=0x128d waiting for monitor
entry [0x2e188000..0x2e189e20]
java.lang.Thread.State: BLOCKED (on object monitor)
at
org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
- waiting to lock <0x45d49d88> (a
org.apache.lucene.store.FSDirectory$FSIndexInput)
at
org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
at
org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
at org.apache.lucene.store.IndexInput.readVLong(IndexInput.java:96)
at
org.apache.lucene.index.MultiLevelSkipListReader.loadSkipLevels(MultiLevelSkipListReader.java:196)
at
org.apache.lucene.index.MultiLevelSkipListReader.skipTo(MultiLevelSkipListReader.java:97)
at org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:164)
at in.verse.search.query.spans.TermSpans.skipTo(TermSpans.java:85)
at in.verse.search.query.spans.SpanScorer.skipTo(SpanScorer.java:70)
at
org.apache.lucene.search.VConjunctionScorer.doNext(VConjunctionScorer.java:78)
at
org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:71)
at org.apache.lucene.search.VBooleanScorer2.next(VBooleanScorer2.java:456)
at
org.apache.lucene.search.VConjunctionScorer.init(VConjunctionScorer.java:136)
at
org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:65)
at org.apache.lucene.search.VBooleanScorer2.score(VBooleanScorer2.java:412)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:146)
at
org.apache.lucene.search.ParallelMultiSearcher.search(ParallelMultiSearcher.java:173)
at org.apache.lucene.search.Searcher.search(Searcher.java:118)
at org.apache.lucene.search.Searcher.search(Searcher.java:97)
at VSearcher$VerticalSearcher$1.call(VSearcher.java:500)
at VSearcher$VerticalSearcher$1.call(VSearcher.java:491)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at
java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:1737)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:767)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:658)
at
java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorService.java:200)
at
in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:542)
at
in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:476)
at in.verse.search.VSearcher$VerticalSearcher.search(VSearcher.java:384)
at in.verse.search.VSearcher.search(VSearcher.java:187)
at
in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:166)
at
in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:255)
at
org.apache.jsp.jobs.mobile.airtel.xhtml.search_jsp._jspService(search_jsp.java:869)
at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at
org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:393)
at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
at
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:433)
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263)
at
org.apache.coyote.http11.Http11AprProcessor.process(Http11AprProcessor.java:852)
at
org.apache.coyote.http11.Http11AprProtocol$Http11ConnectionHandler.process(Http11AprProtocol.java:584)
at org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1508)
at java.lang.Thread.run(Thread.java:619)
--
View this message in context: http://www.nabble.com/Tomcat-Threads-are-BLOCKED-after-some-time-tp22308385p22308385.html
Sent from the Lucene - Java Users mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
RE: Tomcat Threads are BLOCKED after some time
Posted by Uwe Schindler <uw...@thetaphi.de>.
Does this also happen, if you not use ParallelMultiSearcher?
Can you replace by a normal MultiSearcher or alternatively a simple
IndexSearcher on a MultiReader?
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: damu_verse [mailto:damoder.reddy@gmail.com]
> Sent: Tuesday, March 03, 2009 2:13 PM
> To: java-user@lucene.apache.org
> Subject: Tomcat Threads are BLOCKED after some time
>
>
> Hi All,
>
> We Have used Lucene as our Search Engine and all our applications
> are deployed onto tomcat and running with thread pool size of 200.
>
> Java Version - 1.6.0-rc
> Lucene Version - 2.3.2
> Tomcat Version - 6.0.14
> OS - Red Hat Enterprise Linux ES release 4 (Nahant Update 5)
> kernel - 2.6.9-55.0.2.ELsmp
> RAM - 4 GB
> Tomcat Memory - 1.5 GB
> Index Size - 2 GB
>
>
> After 10-12 hrs of tomcat running, tomcat
> becomes irresponsive. After doing core dump of tomcat process We observed
> that all tomcat threads are blocked (Thread-pool size-200). none of the
> tomcat threads are in runnable state.
>
> each thread at the time of the core dump is in BLOCKED state...The
> following
> are the stack trace of blocked.
>
> "MultiSearcher thread #3" daemon prio=10 tid=0x337ddc00 nid=0x4827 waiting
> for monitor entry [0x2f2f0000..0x2f2f0ea0]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(Compo
> undFileReader.java:235)
> - waiting to lock <0x45d49d88> (a
> org.apache.lucene.store.FSDirectory$FSIndexInput)
> at
> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:
> 152)
> at
> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.jav
> a:38)
> at org.apache.lucene.store.IndexInput.readVInt(IndexInput.java:76)
> at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:63)
> at
> org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:123)
> at
> org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:154)
> at
> org.apache.lucene.index.TermInfosReader.scanEnum(TermInfosReader.java:223)
> at
> org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:217)
> at
> org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:54)
> at
> org.apache.lucene.index.IndexReader.termDocs(IndexReader.java:668)
> at
> org.apache.lucene.search.ConstantScoreTermQuery$TermWeight.scorer(Constant
> ScoreTermQuery.java:63)
> at
> org.apache.lucene.search.VBooleanQuery$BooleanWeight.scorer(VBooleanQuery.
> java:276)
> at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.ja
> va:232)
> at
> org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:143)
> at
> org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:124)
> at
> org.apache.lucene.search.MultiSearcherThread.run(ParallelMultiSearcher.jav
> a:250)
>
>
>
> "http-8080-194" daemon prio=10 tid=0x08927800 nid=0x128d waiting for
> monitor
> entry [0x2e188000..0x2e189e20]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(Compo
> undFileReader.java:235)
> - waiting to lock <0x45d49d88> (a
> org.apache.lucene.store.FSDirectory$FSIndexInput)
> at
> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:
> 152)
> at
> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.jav
> a:38)
> at org.apache.lucene.store.IndexInput.readVLong(IndexInput.java:96)
> at
> org.apache.lucene.index.MultiLevelSkipListReader.loadSkipLevels(MultiLevel
> SkipListReader.java:196)
> at
> org.apache.lucene.index.MultiLevelSkipListReader.skipTo(MultiLevelSkipList
> Reader.java:97)
> at
> org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:164)
> at in.verse.search.query.spans.TermSpans.skipTo(TermSpans.java:85)
> at in.verse.search.query.spans.SpanScorer.skipTo(SpanScorer.java:70)
> at
> org.apache.lucene.search.VConjunctionScorer.doNext(VConjunctionScorer.java
> :78)
> at
> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:7
> 1)
> at
> org.apache.lucene.search.VBooleanScorer2.next(VBooleanScorer2.java:456)
> at
> org.apache.lucene.search.VConjunctionScorer.init(VConjunctionScorer.java:1
> 36)
> at
> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:6
> 5)
> at
> org.apache.lucene.search.VBooleanScorer2.score(VBooleanScorer2.java:412)
> at
> org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:146)
> at
> org.apache.lucene.search.ParallelMultiSearcher.search(ParallelMultiSearche
> r.java:173)
> at org.apache.lucene.search.Searcher.search(Searcher.java:118)
> at org.apache.lucene.search.Searcher.search(Searcher.java:97)
> at VSearcher$VerticalSearcher$1.call(VSearcher.java:500)
> at VSearcher$VerticalSearcher$1.call(VSearcher.java:491)
> at
> java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
> at java.util.concurrent.FutureTask.run(FutureTask.java:138)
> at
> java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution
> (ThreadPoolExecutor.java:1737)
> at
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:767
> )
> at
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:65
> 8)
> at
> java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorSer
> vice.java:200)
> at
> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:5
> 42)
> at
> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:4
> 76)
> at
> in.verse.search.VSearcher$VerticalSearcher.search(VSearcher.java:384)
> at in.verse.search.VSearcher.search(VSearcher.java:187)
> at
> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:166)
> at
> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:255)
> at
> org.apache.jsp.jobs.mobile.airtel.xhtml.search_jsp._jspService(search_jsp.
> java:869)
> at
> org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
> at
> org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java
> :393)
> at
> org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
> at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Applicati
> onFilterChain.java:290)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilter
> Chain.java:206)
> at
> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.
> java:233)
> at
> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.
> java:175)
> at
> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBa
> se.java:433)
> at
> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:1
> 28)
> at
> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:1
> 02)
> at
> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.ja
> va:109)
> at
> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263
> )
> at
> org.apache.coyote.http11.Http11AprProcessor.process(Http11AprProcessor.jav
> a:852)
> at
> org.apache.coyote.http11.Http11AprProtocol$Http11ConnectionHandler.process
> (Http11AprProtocol.java:584)
> at
> org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1508)
> at java.lang.Thread.run(Thread.java:619)
> --
> View this message in context: http://www.nabble.com/Tomcat-Threads-are-
> BLOCKED-after-some-time-tp22308385p22308385.html
> Sent from the Lucene - Java Users mailing list archive at Nabble.com.
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> For additional commands, e-mail: java-user-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Tomcat Threads are BLOCKED after some time
Posted by Yonik Seeley <yo...@lucidimagination.com>.
On Fri, Mar 6, 2009 at 5:43 AM, damu_verse <da...@gmail.com> wrote:
> We have tried with the Lucene-2.4.0 also (JVM not changed) ..
> But still threads are blocking..Not able to find the root cause...
What is the *full* thread dump?
Some threads blocking is fine and normal - there isn't necessarily
anything wrong based on the thread dumps you have provided so far.
-Yonik
http://www.lucidimagination.com
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Tomcat Threads are BLOCKED after some time
Posted by damu_verse <da...@gmail.com>.
Hi Yonik
We have tried with the Lucene-2.4.0 also (JVM not changed) ..
But still threads are blocking..Not able to find the root cause...
thanks & regards
-damu
Yonik Seeley-2 wrote:
>
> Hmmm, if this is some sort of deadlock, we may need a thread dump of
> all of the threads.
> Does your app with Lucene 2.4 have the same problem?
> Is it possible to try a newer JVM version (like 1.6.0_12)?
>
> -Yonik
> http://www.lucidimagination.com
>
>
> On Tue, Mar 3, 2009 at 8:13 AM, damu_verse <da...@gmail.com>
> wrote:
>>
>> Hi All,
>>
>> We Have used Lucene as our Search Engine and all our applications
>> are deployed onto tomcat and running with thread pool size of 200.
>>
>> Java Version - 1.6.0-rc
>> Lucene Version - 2.3.2
>> Tomcat Version - 6.0.14
>> OS - Red Hat Enterprise Linux ES release 4 (Nahant Update 5)
>> kernel - 2.6.9-55.0.2.ELsmp
>> RAM - 4 GB
>> Tomcat Memory - 1.5 GB
>> Index Size - 2 GB
>>
>>
>> After 10-12 hrs of tomcat running, tomcat
>> becomes irresponsive. After doing core dump of tomcat process We observed
>> that all tomcat threads are blocked (Thread-pool size-200). none of the
>> tomcat threads are in runnable state.
>>
>> each thread at the time of the core dump is in BLOCKED state...The
>> following
>> are the stack trace of blocked.
>>
>> "MultiSearcher thread #3" daemon prio=10 tid=0x337ddc00 nid=0x4827
>> waiting
>> for monitor entry [0x2f2f0000..0x2f2f0ea0]
>> java.lang.Thread.State: BLOCKED (on object monitor)
>> at
>> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
>> - waiting to lock <0x45d49d88> (a
>> org.apache.lucene.store.FSDirectory$FSIndexInput)
>> at
>> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
>> at
>> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
>> at org.apache.lucene.store.IndexInput.readVInt(IndexInput.java:76)
>> at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:63)
>> at
>> org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:123)
>> at
>> org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:154)
>> at
>> org.apache.lucene.index.TermInfosReader.scanEnum(TermInfosReader.java:223)
>> at
>> org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:217)
>> at
>> org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:54)
>> at
>> org.apache.lucene.index.IndexReader.termDocs(IndexReader.java:668)
>> at
>> org.apache.lucene.search.ConstantScoreTermQuery$TermWeight.scorer(ConstantScoreTermQuery.java:63)
>> at
>> org.apache.lucene.search.VBooleanQuery$BooleanWeight.scorer(VBooleanQuery.java:276)
>> at
>> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:232)
>> at
>> org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:143)
>> at
>> org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:124)
>> at
>> org.apache.lucene.search.MultiSearcherThread.run(ParallelMultiSearcher.java:250)
>>
>>
>>
>> "http-8080-194" daemon prio=10 tid=0x08927800 nid=0x128d waiting for
>> monitor
>> entry [0x2e188000..0x2e189e20]
>> java.lang.Thread.State: BLOCKED (on object monitor)
>> at
>> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
>> - waiting to lock <0x45d49d88> (a
>> org.apache.lucene.store.FSDirectory$FSIndexInput)
>> at
>> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
>> at
>> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
>> at
>> org.apache.lucene.store.IndexInput.readVLong(IndexInput.java:96)
>> at
>> org.apache.lucene.index.MultiLevelSkipListReader.loadSkipLevels(MultiLevelSkipListReader.java:196)
>> at
>> org.apache.lucene.index.MultiLevelSkipListReader.skipTo(MultiLevelSkipListReader.java:97)
>> at
>> org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:164)
>> at in.verse.search.query.spans.TermSpans.skipTo(TermSpans.java:85)
>> at
>> in.verse.search.query.spans.SpanScorer.skipTo(SpanScorer.java:70)
>> at
>> org.apache.lucene.search.VConjunctionScorer.doNext(VConjunctionScorer.java:78)
>> at
>> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:71)
>> at
>> org.apache.lucene.search.VBooleanScorer2.next(VBooleanScorer2.java:456)
>> at
>> org.apache.lucene.search.VConjunctionScorer.init(VConjunctionScorer.java:136)
>> at
>> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:65)
>> at
>> org.apache.lucene.search.VBooleanScorer2.score(VBooleanScorer2.java:412)
>> at
>> org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:146)
>> at
>> org.apache.lucene.search.ParallelMultiSearcher.search(ParallelMultiSearcher.java:173)
>> at org.apache.lucene.search.Searcher.search(Searcher.java:118)
>> at org.apache.lucene.search.Searcher.search(Searcher.java:97)
>> at VSearcher$VerticalSearcher$1.call(VSearcher.java:500)
>> at VSearcher$VerticalSearcher$1.call(VSearcher.java:491)
>> at
>> java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
>> at java.util.concurrent.FutureTask.run(FutureTask.java:138)
>> at
>> java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:1737)
>> at
>> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:767)
>> at
>> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:658)
>> at
>> java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorService.java:200)
>> at
>> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:542)
>> at
>> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:476)
>> at
>> in.verse.search.VSearcher$VerticalSearcher.search(VSearcher.java:384)
>> at in.verse.search.VSearcher.search(VSearcher.java:187)
>> at
>> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:166)
>> at
>> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:255)
>> at
>> org.apache.jsp.jobs.mobile.airtel.xhtml.search_jsp._jspService(search_jsp.java:869)
>> at
>> org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
>> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
>> at
>> org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:393)
>> at
>> org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
>> at
>> org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
>> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
>> at
>> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
>> at
>> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
>> at
>> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
>> at
>> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
>> at
>> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:433)
>> at
>> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
>> at
>> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
>> at
>> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
>> at
>> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263)
>> at
>> org.apache.coyote.http11.Http11AprProcessor.process(Http11AprProcessor.java:852)
>> at
>> org.apache.coyote.http11.Http11AprProtocol$Http11ConnectionHandler.process(Http11AprProtocol.java:584)
>> at
>> org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1508)
>> at java.lang.Thread.run(Thread.java:619)
>> --
>> View this message in context:
>> http://www.nabble.com/Tomcat-Threads-are-BLOCKED-after-some-time-tp22308385p22308385.html
>> Sent from the Lucene - Java Users mailing list archive at Nabble.com.
>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: java-user-help@lucene.apache.org
>>
>>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> For additional commands, e-mail: java-user-help@lucene.apache.org
>
>
>
--
View this message in context: http://www.nabble.com/Tomcat-Threads-are-BLOCKED-after-some-time-tp22308385p22369462.html
Sent from the Lucene - Java Users mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Tomcat Threads are BLOCKED after some time
Posted by Yonik Seeley <yo...@lucidimagination.com>.
Hmmm, if this is some sort of deadlock, we may need a thread dump of
all of the threads.
Does your app with Lucene 2.4 have the same problem?
Is it possible to try a newer JVM version (like 1.6.0_12)?
-Yonik
http://www.lucidimagination.com
On Tue, Mar 3, 2009 at 8:13 AM, damu_verse <da...@gmail.com> wrote:
>
> Hi All,
>
> We Have used Lucene as our Search Engine and all our applications
> are deployed onto tomcat and running with thread pool size of 200.
>
> Java Version - 1.6.0-rc
> Lucene Version - 2.3.2
> Tomcat Version - 6.0.14
> OS - Red Hat Enterprise Linux ES release 4 (Nahant Update 5)
> kernel - 2.6.9-55.0.2.ELsmp
> RAM - 4 GB
> Tomcat Memory - 1.5 GB
> Index Size - 2 GB
>
>
> After 10-12 hrs of tomcat running, tomcat
> becomes irresponsive. After doing core dump of tomcat process We observed
> that all tomcat threads are blocked (Thread-pool size-200). none of the
> tomcat threads are in runnable state.
>
> each thread at the time of the core dump is in BLOCKED state...The following
> are the stack trace of blocked.
>
> "MultiSearcher thread #3" daemon prio=10 tid=0x337ddc00 nid=0x4827 waiting
> for monitor entry [0x2f2f0000..0x2f2f0ea0]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
> - waiting to lock <0x45d49d88> (a
> org.apache.lucene.store.FSDirectory$FSIndexInput)
> at
> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
> at
> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
> at org.apache.lucene.store.IndexInput.readVInt(IndexInput.java:76)
> at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:63)
> at org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:123)
> at org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:154)
> at
> org.apache.lucene.index.TermInfosReader.scanEnum(TermInfosReader.java:223)
> at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:217)
> at org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:54)
> at org.apache.lucene.index.IndexReader.termDocs(IndexReader.java:668)
> at
> org.apache.lucene.search.ConstantScoreTermQuery$TermWeight.scorer(ConstantScoreTermQuery.java:63)
> at
> org.apache.lucene.search.VBooleanQuery$BooleanWeight.scorer(VBooleanQuery.java:276)
> at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:232)
> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:143)
> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:124)
> at
> org.apache.lucene.search.MultiSearcherThread.run(ParallelMultiSearcher.java:250)
>
>
>
> "http-8080-194" daemon prio=10 tid=0x08927800 nid=0x128d waiting for monitor
> entry [0x2e188000..0x2e189e20]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
> - waiting to lock <0x45d49d88> (a
> org.apache.lucene.store.FSDirectory$FSIndexInput)
> at
> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
> at
> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
> at org.apache.lucene.store.IndexInput.readVLong(IndexInput.java:96)
> at
> org.apache.lucene.index.MultiLevelSkipListReader.loadSkipLevels(MultiLevelSkipListReader.java:196)
> at
> org.apache.lucene.index.MultiLevelSkipListReader.skipTo(MultiLevelSkipListReader.java:97)
> at org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:164)
> at in.verse.search.query.spans.TermSpans.skipTo(TermSpans.java:85)
> at in.verse.search.query.spans.SpanScorer.skipTo(SpanScorer.java:70)
> at
> org.apache.lucene.search.VConjunctionScorer.doNext(VConjunctionScorer.java:78)
> at
> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:71)
> at org.apache.lucene.search.VBooleanScorer2.next(VBooleanScorer2.java:456)
> at
> org.apache.lucene.search.VConjunctionScorer.init(VConjunctionScorer.java:136)
> at
> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:65)
> at org.apache.lucene.search.VBooleanScorer2.score(VBooleanScorer2.java:412)
> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:146)
> at
> org.apache.lucene.search.ParallelMultiSearcher.search(ParallelMultiSearcher.java:173)
> at org.apache.lucene.search.Searcher.search(Searcher.java:118)
> at org.apache.lucene.search.Searcher.search(Searcher.java:97)
> at VSearcher$VerticalSearcher$1.call(VSearcher.java:500)
> at VSearcher$VerticalSearcher$1.call(VSearcher.java:491)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
> at java.util.concurrent.FutureTask.run(FutureTask.java:138)
> at
> java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:1737)
> at
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:767)
> at
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:658)
> at
> java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorService.java:200)
> at
> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:542)
> at
> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:476)
> at in.verse.search.VSearcher$VerticalSearcher.search(VSearcher.java:384)
> at in.verse.search.VSearcher.search(VSearcher.java:187)
> at
> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:166)
> at
> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:255)
> at
> org.apache.jsp.jobs.mobile.airtel.xhtml.search_jsp._jspService(search_jsp.java:869)
> at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
> at
> org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:393)
> at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
> at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
> at
> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
> at
> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
> at
> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:433)
> at
> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
> at
> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
> at
> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
> at
> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263)
> at
> org.apache.coyote.http11.Http11AprProcessor.process(Http11AprProcessor.java:852)
> at
> org.apache.coyote.http11.Http11AprProtocol$Http11ConnectionHandler.process(Http11AprProtocol.java:584)
> at org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1508)
> at java.lang.Thread.run(Thread.java:619)
> --
> View this message in context: http://www.nabble.com/Tomcat-Threads-are-BLOCKED-after-some-time-tp22308385p22308385.html
> Sent from the Lucene - Java Users mailing list archive at Nabble.com.
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> For additional commands, e-mail: java-user-help@lucene.apache.org
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Tomcat Threads are BLOCKED after some time
Posted by damu_verse <da...@gmail.com>.
We have observed the same issue for 3 days in row ... And it has become
really urgent... If any body has an idea about this issue please help me.
Thanks in Advance
-damu
damu_verse wrote:
>
> Hi All,
>
> We Have used Lucene as our Search Engine and all our applications
> are deployed onto tomcat and running with thread pool size of 200.
>
> Java Version - 1.6.0-rc
> Lucene Version - 2.3.2
> Tomcat Version - 6.0.14
> OS - Red Hat Enterprise Linux ES release 4 (Nahant Update 5)
> kernel - 2.6.9-55.0.2.ELsmp
> RAM - 4 GB
> Tomcat Memory - 1.5 GB
> Index Size - 2 GB
>
>
> After 10-12 hrs of tomcat running, tomcat
> becomes irresponsive. After doing core dump of tomcat process We observed
> that all tomcat threads are blocked (Thread-pool size-200). none of the
> tomcat threads are in runnable state.
>
> each thread at the time of the core dump is in BLOCKED state...The
> following are the stack trace of blocked.
>
> "MultiSearcher thread #3" daemon prio=10 tid=0x337ddc00 nid=0x4827 waiting
> for monitor entry [0x2f2f0000..0x2f2f0ea0]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
> - waiting to lock <0x45d49d88> (a
> org.apache.lucene.store.FSDirectory$FSIndexInput)
> at
> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
> at
> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
> at org.apache.lucene.store.IndexInput.readVInt(IndexInput.java:76)
> at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:63)
> at org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:123)
> at
> org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:154)
> at
> org.apache.lucene.index.TermInfosReader.scanEnum(TermInfosReader.java:223)
> at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:217)
> at org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:54)
> at org.apache.lucene.index.IndexReader.termDocs(IndexReader.java:668)
> at
> org.apache.lucene.search.ConstantScoreTermQuery$TermWeight.scorer(ConstantScoreTermQuery.java:63)
> at
> org.apache.lucene.search.VBooleanQuery$BooleanWeight.scorer(VBooleanQuery.java:276)
> at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:232)
> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:143)
> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:124)
> at
> org.apache.lucene.search.MultiSearcherThread.run(ParallelMultiSearcher.java:250)
>
>
>
> "http-8080-194" daemon prio=10 tid=0x08927800 nid=0x128d waiting for
> monitor entry [0x2e188000..0x2e189e20]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
> - waiting to lock <0x45d49d88> (a
> org.apache.lucene.store.FSDirectory$FSIndexInput)
> at
> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
> at
> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
> at org.apache.lucene.store.IndexInput.readVLong(IndexInput.java:96)
> at
> org.apache.lucene.index.MultiLevelSkipListReader.loadSkipLevels(MultiLevelSkipListReader.java:196)
> at
> org.apache.lucene.index.MultiLevelSkipListReader.skipTo(MultiLevelSkipListReader.java:97)
> at
> org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:164)
> at in.verse.search.query.spans.TermSpans.skipTo(TermSpans.java:85)
> at in.verse.search.query.spans.SpanScorer.skipTo(SpanScorer.java:70)
> at
> org.apache.lucene.search.VConjunctionScorer.doNext(VConjunctionScorer.java:78)
> at
> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:71)
> at
> org.apache.lucene.search.VBooleanScorer2.next(VBooleanScorer2.java:456)
> at
> org.apache.lucene.search.VConjunctionScorer.init(VConjunctionScorer.java:136)
> at
> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:65)
> at
> org.apache.lucene.search.VBooleanScorer2.score(VBooleanScorer2.java:412)
> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:146)
> at
> org.apache.lucene.search.ParallelMultiSearcher.search(ParallelMultiSearcher.java:173)
> at org.apache.lucene.search.Searcher.search(Searcher.java:118)
> at org.apache.lucene.search.Searcher.search(Searcher.java:97)
> at VSearcher$VerticalSearcher$1.call(VSearcher.java:500)
> at VSearcher$VerticalSearcher$1.call(VSearcher.java:491)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
> at java.util.concurrent.FutureTask.run(FutureTask.java:138)
> at
> java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:1737)
> at
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:767)
> at
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:658)
> at
> java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorService.java:200)
> at
> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:542)
> at
> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:476)
> at in.verse.search.VSearcher$VerticalSearcher.search(VSearcher.java:384)
> at in.verse.search.VSearcher.search(VSearcher.java:187)
> at
> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:166)
> at
> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:255)
> at
> org.apache.jsp.jobs.mobile.airtel.xhtml.search_jsp._jspService(search_jsp.java:869)
> at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
> at
> org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:393)
> at
> org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
> at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
> at
> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
> at
> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
> at
> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:433)
> at
> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
> at
> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
> at
> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
> at
> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263)
> at
> org.apache.coyote.http11.Http11AprProcessor.process(Http11AprProcessor.java:852)
> at
> org.apache.coyote.http11.Http11AprProtocol$Http11ConnectionHandler.process(Http11AprProtocol.java:584)
> at
> org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1508)
> at java.lang.Thread.run(Thread.java:619)
>
--
View this message in context: http://www.nabble.com/Tomcat-Threads-are-BLOCKED-after-some-time-tp22308385p22308511.html
Sent from the Lucene - Java Users mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Tomcat Threads are BLOCKED after some time
Posted by Varun Dhussa <va...@mapmyindia.com>.
Hi,
I think it might be a case of the allowed open files at the OS. Try
setting a higher ulimit and run the program. Also, what are the gc
parameters you have set on the jvm?
Regards
Varun Dhussa
Product Architect
CE InfoSystems (P) Ltd
http://www.mapmyindia.com
damu_verse wrote:
> Hi Thanx for the reply..
> we have not tested this against the versions
> (both java-1.6.12 and lucene-2.4) mentioned and more over we can not move to
> those verions right away... So we need a solution for this particular
> version only..
>
> thanx & regards
> damu
>
> damu_verse wrote:
>
>> Hi All,
>>
>> We Have used Lucene as our Search Engine and all our applications
>> are deployed onto tomcat and running with thread pool size of 200.
>>
>> Java Version - 1.6.0-rc
>> Lucene Version - 2.3.2
>> Tomcat Version - 6.0.14
>> OS - Red Hat Enterprise Linux ES release 4 (Nahant Update 5)
>> kernel - 2.6.9-55.0.2.ELsmp
>> RAM - 4 GB
>> Tomcat Memory - 1.5 GB
>> Index Size - 2 GB
>>
>>
>> After 10-12 hrs of tomcat running, tomcat
>> becomes irresponsive. After doing core dump of tomcat process We observed
>> that all tomcat threads are blocked (Thread-pool size-200). none of the
>> tomcat threads are in runnable state.
>>
>> each thread at the time of the core dump is in BLOCKED state...The
>> following are the stack trace of blocked.
>>
>> "MultiSearcher thread #3" daemon prio=10 tid=0x337ddc00 nid=0x4827 waiting
>> for monitor entry [0x2f2f0000..0x2f2f0ea0]
>> java.lang.Thread.State: BLOCKED (on object monitor)
>> at
>> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
>> - waiting to lock <0x45d49d88> (a
>> org.apache.lucene.store.FSDirectory$FSIndexInput)
>> at
>> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
>> at
>> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
>> at org.apache.lucene.store.IndexInput.readVInt(IndexInput.java:76)
>> at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:63)
>> at org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:123)
>> at
>> org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:154)
>> at
>> org.apache.lucene.index.TermInfosReader.scanEnum(TermInfosReader.java:223)
>> at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:217)
>> at org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:54)
>> at org.apache.lucene.index.IndexReader.termDocs(IndexReader.java:668)
>> at
>> org.apache.lucene.search.ConstantScoreTermQuery$TermWeight.scorer(ConstantScoreTermQuery.java:63)
>> at
>> org.apache.lucene.search.VBooleanQuery$BooleanWeight.scorer(VBooleanQuery.java:276)
>> at
>> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:232)
>> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:143)
>> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:124)
>> at
>> org.apache.lucene.search.MultiSearcherThread.run(ParallelMultiSearcher.java:250)
>>
>>
>>
>> "http-8080-194" daemon prio=10 tid=0x08927800 nid=0x128d waiting for
>> monitor entry [0x2e188000..0x2e189e20]
>> java.lang.Thread.State: BLOCKED (on object monitor)
>> at
>> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
>> - waiting to lock <0x45d49d88> (a
>> org.apache.lucene.store.FSDirectory$FSIndexInput)
>> at
>> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
>> at
>> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
>> at org.apache.lucene.store.IndexInput.readVLong(IndexInput.java:96)
>> at
>> org.apache.lucene.index.MultiLevelSkipListReader.loadSkipLevels(MultiLevelSkipListReader.java:196)
>> at
>> org.apache.lucene.index.MultiLevelSkipListReader.skipTo(MultiLevelSkipListReader.java:97)
>> at
>> org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:164)
>> at in.verse.search.query.spans.TermSpans.skipTo(TermSpans.java:85)
>> at in.verse.search.query.spans.SpanScorer.skipTo(SpanScorer.java:70)
>> at
>> org.apache.lucene.search.VConjunctionScorer.doNext(VConjunctionScorer.java:78)
>> at
>> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:71)
>> at
>> org.apache.lucene.search.VBooleanScorer2.next(VBooleanScorer2.java:456)
>> at
>> org.apache.lucene.search.VConjunctionScorer.init(VConjunctionScorer.java:136)
>> at
>> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:65)
>> at
>> org.apache.lucene.search.VBooleanScorer2.score(VBooleanScorer2.java:412)
>> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:146)
>> at
>> org.apache.lucene.search.ParallelMultiSearcher.search(ParallelMultiSearcher.java:173)
>> at org.apache.lucene.search.Searcher.search(Searcher.java:118)
>> at org.apache.lucene.search.Searcher.search(Searcher.java:97)
>> at VSearcher$VerticalSearcher$1.call(VSearcher.java:500)
>> at VSearcher$VerticalSearcher$1.call(VSearcher.java:491)
>> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
>> at java.util.concurrent.FutureTask.run(FutureTask.java:138)
>> at
>> java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:1737)
>> at
>> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:767)
>> at
>> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:658)
>> at
>> java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorService.java:200)
>> at
>> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:542)
>> at
>> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:476)
>> at in.verse.search.VSearcher$VerticalSearcher.search(VSearcher.java:384)
>> at in.verse.search.VSearcher.search(VSearcher.java:187)
>> at
>> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:166)
>> at
>> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:255)
>> at
>> org.apache.jsp.jobs.mobile.airtel.xhtml.search_jsp._jspService(search_jsp.java:869)
>> at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
>> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
>> at
>> org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:393)
>> at
>> org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
>> at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
>> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
>> at
>> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
>> at
>> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
>> at
>> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
>> at
>> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
>> at
>> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:433)
>> at
>> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
>> at
>> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
>> at
>> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
>> at
>> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263)
>> at
>> org.apache.coyote.http11.Http11AprProcessor.process(Http11AprProcessor.java:852)
>> at
>> org.apache.coyote.http11.Http11AprProtocol$Http11ConnectionHandler.process(Http11AprProtocol.java:584)
>> at
>> org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1508)
>> at java.lang.Thread.run(Thread.java:619)
>>
>>
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Tomcat Threads are BLOCKED after some time
Posted by damu_verse <da...@gmail.com>.
Hi Thanx for the reply..
we have not tested this against the versions
(both java-1.6.12 and lucene-2.4) mentioned and more over we can not move to
those verions right away... So we need a solution for this particular
version only..
thanx & regards
damu
damu_verse wrote:
>
> Hi All,
>
> We Have used Lucene as our Search Engine and all our applications
> are deployed onto tomcat and running with thread pool size of 200.
>
> Java Version - 1.6.0-rc
> Lucene Version - 2.3.2
> Tomcat Version - 6.0.14
> OS - Red Hat Enterprise Linux ES release 4 (Nahant Update 5)
> kernel - 2.6.9-55.0.2.ELsmp
> RAM - 4 GB
> Tomcat Memory - 1.5 GB
> Index Size - 2 GB
>
>
> After 10-12 hrs of tomcat running, tomcat
> becomes irresponsive. After doing core dump of tomcat process We observed
> that all tomcat threads are blocked (Thread-pool size-200). none of the
> tomcat threads are in runnable state.
>
> each thread at the time of the core dump is in BLOCKED state...The
> following are the stack trace of blocked.
>
> "MultiSearcher thread #3" daemon prio=10 tid=0x337ddc00 nid=0x4827 waiting
> for monitor entry [0x2f2f0000..0x2f2f0ea0]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
> - waiting to lock <0x45d49d88> (a
> org.apache.lucene.store.FSDirectory$FSIndexInput)
> at
> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
> at
> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
> at org.apache.lucene.store.IndexInput.readVInt(IndexInput.java:76)
> at org.apache.lucene.index.TermBuffer.read(TermBuffer.java:63)
> at org.apache.lucene.index.SegmentTermEnum.next(SegmentTermEnum.java:123)
> at
> org.apache.lucene.index.SegmentTermEnum.scanTo(SegmentTermEnum.java:154)
> at
> org.apache.lucene.index.TermInfosReader.scanEnum(TermInfosReader.java:223)
> at org.apache.lucene.index.TermInfosReader.get(TermInfosReader.java:217)
> at org.apache.lucene.index.SegmentTermDocs.seek(SegmentTermDocs.java:54)
> at org.apache.lucene.index.IndexReader.termDocs(IndexReader.java:668)
> at
> org.apache.lucene.search.ConstantScoreTermQuery$TermWeight.scorer(ConstantScoreTermQuery.java:63)
> at
> org.apache.lucene.search.VBooleanQuery$BooleanWeight.scorer(VBooleanQuery.java:276)
> at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:232)
> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:143)
> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:124)
> at
> org.apache.lucene.search.MultiSearcherThread.run(ParallelMultiSearcher.java:250)
>
>
>
> "http-8080-194" daemon prio=10 tid=0x08927800 nid=0x128d waiting for
> monitor entry [0x2e188000..0x2e189e20]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.lucene.index.CompoundFileReader$CSIndexInput.readInternal(CompoundFileReader.java:235)
> - waiting to lock <0x45d49d88> (a
> org.apache.lucene.store.FSDirectory$FSIndexInput)
> at
> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:152)
> at
> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:38)
> at org.apache.lucene.store.IndexInput.readVLong(IndexInput.java:96)
> at
> org.apache.lucene.index.MultiLevelSkipListReader.loadSkipLevels(MultiLevelSkipListReader.java:196)
> at
> org.apache.lucene.index.MultiLevelSkipListReader.skipTo(MultiLevelSkipListReader.java:97)
> at
> org.apache.lucene.index.SegmentTermDocs.skipTo(SegmentTermDocs.java:164)
> at in.verse.search.query.spans.TermSpans.skipTo(TermSpans.java:85)
> at in.verse.search.query.spans.SpanScorer.skipTo(SpanScorer.java:70)
> at
> org.apache.lucene.search.VConjunctionScorer.doNext(VConjunctionScorer.java:78)
> at
> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:71)
> at
> org.apache.lucene.search.VBooleanScorer2.next(VBooleanScorer2.java:456)
> at
> org.apache.lucene.search.VConjunctionScorer.init(VConjunctionScorer.java:136)
> at
> org.apache.lucene.search.VConjunctionScorer.next(VConjunctionScorer.java:65)
> at
> org.apache.lucene.search.VBooleanScorer2.score(VBooleanScorer2.java:412)
> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:146)
> at
> org.apache.lucene.search.ParallelMultiSearcher.search(ParallelMultiSearcher.java:173)
> at org.apache.lucene.search.Searcher.search(Searcher.java:118)
> at org.apache.lucene.search.Searcher.search(Searcher.java:97)
> at VSearcher$VerticalSearcher$1.call(VSearcher.java:500)
> at VSearcher$VerticalSearcher$1.call(VSearcher.java:491)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
> at java.util.concurrent.FutureTask.run(FutureTask.java:138)
> at
> java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:1737)
> at
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:767)
> at
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:658)
> at
> java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorService.java:200)
> at
> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:542)
> at
> in.verse.search.VSearcher$VerticalSearcher.searchInternal(VSearcher.java:476)
> at in.verse.search.VSearcher$VerticalSearcher.search(VSearcher.java:384)
> at in.verse.search.VSearcher.search(VSearcher.java:187)
> at
> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:166)
> at
> in.verse.search.impl.SearchQuery.executeSearchQuery(SearchQuery.java:255)
> at
> org.apache.jsp.jobs.mobile.airtel.xhtml.search_jsp._jspService(search_jsp.java:869)
> at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
> at
> org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:393)
> at
> org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
> at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
> at
> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
> at
> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
> at
> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:433)
> at
> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
> at
> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
> at
> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
> at
> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263)
> at
> org.apache.coyote.http11.Http11AprProcessor.process(Http11AprProcessor.java:852)
> at
> org.apache.coyote.http11.Http11AprProtocol$Http11ConnectionHandler.process(Http11AprProtocol.java:584)
> at
> org.apache.tomcat.util.net.AprEndpoint$Worker.run(AprEndpoint.java:1508)
> at java.lang.Thread.run(Thread.java:619)
>
--
View this message in context: http://www.nabble.com/Tomcat-Threads-are-BLOCKED-after-some-time-tp22308385p22309637.html
Sent from the Lucene - Java Users mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org