You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Raja Pothuganti <RP...@competitrack.com> on 2015/10/23 16:17:41 UTC
OOM on solr cloud 5.2.1, does not trigger oom_solr.sh
Hi,
Some times I see OOM happening on replicas,but does not trigger script
oom_solr.sh which was passed in as
-XX:OnOutOfMemoryError=/actualLocation/solr/bin/oom_solr.sh 8091.
These OOM happened while DIH importing data from database. Is this known
issue? is there any quick fix? Sent yesterday day to users group, no
response yet.
Here are stack traces when OOM happened
1)
org.apache.solr.common.SolrException; null:java.lang.RuntimeException:
java.lang.OutOfMemoryError: Java heap space
at
org.apache.solr.servlet.HttpSolrCall.sendError(HttpSolrCall.java:593)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:465)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
:227)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
:196)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandle
r.java:1652)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
3)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.jav
a:223)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.jav
a:1127)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java
:185)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java
:1061)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
1)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHan
dlerCollection.java:215)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection
.java:110)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:
97)
at org.eclipse.jetty.server.Server.handle(Server.java:497)
at
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
at
org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java
:635)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:
555)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Java heap space
2)
org.apache.solr.common.SolrException;
org.apache.solr.common.SolrException: Exception writing document id
R277453962 to the index; possible analysis error.
at
org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.jav
a:167)
at
org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdatePro
cessorFactory.java:69)
at
org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRe
questProcessor.java:51)
at
org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(Dist
ributedUpdateProcessor.java:955)
at
org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(Dist
ributedUpdateProcessor.java:1110)
at
org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(Dist
ributedUpdateProcessor.java:706)
at
org.apache.solr.update.processor.LogUpdateProcessor.processAdd(LogUpdatePro
cessorFactory.java:104)
at
org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:10
1)
at
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterM
ostDocIterator(JavaBinUpdateRequestCodec.java:179)
at
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterat
or(JavaBinUpdateRequestCodec.java:135)
at
org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:241)
at
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedL
ist(JavaBinUpdateRequestCodec.java:121)
at
org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:206)
at
org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:126)
at
org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(Ja
vaBinUpdateRequestCodec.java:186)
at
org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader
.java:111)
at
org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:58)
at
org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.ja
va:98)
at
org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentS
treamHandlerBase.java:74)
at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase
.java:143)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2064)
at
org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:654)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:450)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
:227)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
:196)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandle
r.java:1652)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
3)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.jav
a:223)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.jav
a:1127)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java
:185)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java
:1061)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
1)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHan
dlerCollection.java:215)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection
.java:110)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:
97)
at org.eclipse.jetty.server.Server.handle(Server.java:497)
at
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
at
org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java
:635)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:
555)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.lucene.store.AlreadyClosedException: this
IndexWriter is closed
at
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:718)
at
org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:732)
at
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1359)
at
org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.ja
va:239)
at
org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.jav
a:163)
... 44 more
Caused by: java.lang.OutOfMemoryError: Java heap space
Thanks
Raja
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: OOM on solr cloud 5.2.1, does not trigger oom_solr.sh
Posted by Erick Erickson <er...@gmail.com>.
done: https://issues.apache.org/jira/browse/SOLR-8193
On Fri, Oct 23, 2015 at 8:16 AM, Timothy Potter <th...@gmail.com> wrote:
> Yes, let's create a JIRA ... looks like the OOM is getting wrapped
> which prevents it from propagating correctly to trigger the oom
> script:
>
> org.apache.solr.common.SolrException; null:java.lang.RuntimeException:
> java.lang.OutOfMemoryError: Java heap space
>
> On Fri, Oct 23, 2015 at 8:28 AM, Erick Erickson <er...@gmail.com> wrote:
>> There have been a couple of threads lately discussing that some OOMs
>> are not propagated appropriately and thus don't trigger the OOM killer.
>>
>> Does anyone think this should be a JIRA?
>>
>> On Fri, Oct 23, 2015 at 7:17 AM, Raja Pothuganti
>> <RP...@competitrack.com> wrote:
>>> Hi,
>>>
>>> Some times I see OOM happening on replicas,but does not trigger script
>>> oom_solr.sh which was passed in as
>>> -XX:OnOutOfMemoryError=/actualLocation/solr/bin/oom_solr.sh 8091.
>>>
>>> These OOM happened while DIH importing data from database. Is this known
>>> issue? is there any quick fix? Sent yesterday day to users group, no
>>> response yet.
>>>
>>> Here are stack traces when OOM happened
>>>
>>>
>>> 1)
>>> org.apache.solr.common.SolrException; null:java.lang.RuntimeException:
>>> java.lang.OutOfMemoryError: Java heap space
>>> at
>>> org.apache.solr.servlet.HttpSolrCall.sendError(HttpSolrCall.java:593)
>>> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:465)
>>> at
>>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
>>> :227)
>>> at
>>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
>>> :196)
>>> at
>>> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandle
>>> r.java:1652)
>>> at
>>> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
>>> at
>>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
>>> 3)
>>> at
>>> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
>>> at
>>> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.jav
>>> a:223)
>>> at
>>> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.jav
>>> a:1127)
>>> at
>>> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
>>> at
>>> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java
>>> :185)
>>> at
>>> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java
>>> :1061)
>>> at
>>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
>>> 1)
>>> at
>>> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHan
>>> dlerCollection.java:215)
>>> at
>>> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection
>>> .java:110)
>>> at
>>> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:
>>> 97)
>>> at org.eclipse.jetty.server.Server.handle(Server.java:497)
>>> at
>>> org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
>>> at
>>> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
>>> at
>>> org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
>>> at
>>> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java
>>> :635)
>>> at
>>> org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:
>>> 555)
>>> at java.lang.Thread.run(Thread.java:745)
>>> Caused by: java.lang.OutOfMemoryError: Java heap space
>>>
>>>
>>>
>>> 2)
>>> org.apache.solr.common.SolrException;
>>> org.apache.solr.common.SolrException: Exception writing document id
>>> R277453962 to the index; possible analysis error.
>>> at
>>> org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.jav
>>> a:167)
>>> at
>>> org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdatePro
>>> cessorFactory.java:69)
>>> at
>>> org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRe
>>> questProcessor.java:51)
>>> at
>>> org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(Dist
>>> ributedUpdateProcessor.java:955)
>>> at
>>> org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(Dist
>>> ributedUpdateProcessor.java:1110)
>>> at
>>> org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(Dist
>>> ributedUpdateProcessor.java:706)
>>> at
>>> org.apache.solr.update.processor.LogUpdateProcessor.processAdd(LogUpdatePro
>>> cessorFactory.java:104)
>>> at
>>> org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:10
>>> 1)
>>> at
>>> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterM
>>> ostDocIterator(JavaBinUpdateRequestCodec.java:179)
>>> at
>>> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterat
>>> or(JavaBinUpdateRequestCodec.java:135)
>>> at
>>> org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:241)
>>> at
>>> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedL
>>> ist(JavaBinUpdateRequestCodec.java:121)
>>> at
>>> org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:206)
>>> at
>>> org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:126)
>>> at
>>> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(Ja
>>> vaBinUpdateRequestCodec.java:186)
>>> at
>>> org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader
>>> .java:111)
>>> at
>>> org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:58)
>>> at
>>> org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.ja
>>> va:98)
>>> at
>>> org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentS
>>> treamHandlerBase.java:74)
>>> at
>>> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase
>>> .java:143)
>>> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2064)
>>> at
>>> org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:654)
>>> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:450)
>>> at
>>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
>>> :227)
>>> at
>>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
>>> :196)
>>> at
>>> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandle
>>> r.java:1652)
>>> at
>>> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
>>> at
>>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
>>> 3)
>>> at
>>> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
>>> at
>>> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.jav
>>> a:223)
>>> at
>>> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.jav
>>> a:1127)
>>> at
>>> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
>>> at
>>> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java
>>> :185)
>>> at
>>> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java
>>> :1061)
>>> at
>>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
>>> 1)
>>> at
>>> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHan
>>> dlerCollection.java:215)
>>> at
>>> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection
>>> .java:110)
>>> at
>>> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:
>>> 97)
>>> at org.eclipse.jetty.server.Server.handle(Server.java:497)
>>> at
>>> org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
>>> at
>>> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
>>> at
>>> org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
>>> at
>>> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java
>>> :635)
>>> at
>>> org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:
>>> 555)
>>> at java.lang.Thread.run(Thread.java:745)
>>> Caused by: org.apache.lucene.store.AlreadyClosedException: this
>>> IndexWriter is closed
>>> at
>>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:718)
>>> at
>>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:732)
>>> at
>>> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1359)
>>> at
>>> org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.ja
>>> va:239)
>>> at
>>> org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.jav
>>> a:163)
>>> ... 44 more
>>> Caused by: java.lang.OutOfMemoryError: Java heap space
>>>
>>>
>>>
>>> Thanks
>>> Raja
>>>
>>>
>>> ---------------------------------------------------------------------
>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: OOM on solr cloud 5.2.1, does not trigger oom_solr.sh
Posted by Timothy Potter <th...@gmail.com>.
Yes, let's create a JIRA ... looks like the OOM is getting wrapped
which prevents it from propagating correctly to trigger the oom
script:
org.apache.solr.common.SolrException; null:java.lang.RuntimeException:
java.lang.OutOfMemoryError: Java heap space
On Fri, Oct 23, 2015 at 8:28 AM, Erick Erickson <er...@gmail.com> wrote:
> There have been a couple of threads lately discussing that some OOMs
> are not propagated appropriately and thus don't trigger the OOM killer.
>
> Does anyone think this should be a JIRA?
>
> On Fri, Oct 23, 2015 at 7:17 AM, Raja Pothuganti
> <RP...@competitrack.com> wrote:
>> Hi,
>>
>> Some times I see OOM happening on replicas,but does not trigger script
>> oom_solr.sh which was passed in as
>> -XX:OnOutOfMemoryError=/actualLocation/solr/bin/oom_solr.sh 8091.
>>
>> These OOM happened while DIH importing data from database. Is this known
>> issue? is there any quick fix? Sent yesterday day to users group, no
>> response yet.
>>
>> Here are stack traces when OOM happened
>>
>>
>> 1)
>> org.apache.solr.common.SolrException; null:java.lang.RuntimeException:
>> java.lang.OutOfMemoryError: Java heap space
>> at
>> org.apache.solr.servlet.HttpSolrCall.sendError(HttpSolrCall.java:593)
>> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:465)
>> at
>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
>> :227)
>> at
>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
>> :196)
>> at
>> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandle
>> r.java:1652)
>> at
>> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
>> at
>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
>> 3)
>> at
>> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
>> at
>> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.jav
>> a:223)
>> at
>> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.jav
>> a:1127)
>> at
>> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
>> at
>> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java
>> :185)
>> at
>> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java
>> :1061)
>> at
>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
>> 1)
>> at
>> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHan
>> dlerCollection.java:215)
>> at
>> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection
>> .java:110)
>> at
>> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:
>> 97)
>> at org.eclipse.jetty.server.Server.handle(Server.java:497)
>> at
>> org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
>> at
>> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
>> at
>> org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
>> at
>> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java
>> :635)
>> at
>> org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:
>> 555)
>> at java.lang.Thread.run(Thread.java:745)
>> Caused by: java.lang.OutOfMemoryError: Java heap space
>>
>>
>>
>> 2)
>> org.apache.solr.common.SolrException;
>> org.apache.solr.common.SolrException: Exception writing document id
>> R277453962 to the index; possible analysis error.
>> at
>> org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.jav
>> a:167)
>> at
>> org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdatePro
>> cessorFactory.java:69)
>> at
>> org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRe
>> questProcessor.java:51)
>> at
>> org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(Dist
>> ributedUpdateProcessor.java:955)
>> at
>> org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(Dist
>> ributedUpdateProcessor.java:1110)
>> at
>> org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(Dist
>> ributedUpdateProcessor.java:706)
>> at
>> org.apache.solr.update.processor.LogUpdateProcessor.processAdd(LogUpdatePro
>> cessorFactory.java:104)
>> at
>> org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:10
>> 1)
>> at
>> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterM
>> ostDocIterator(JavaBinUpdateRequestCodec.java:179)
>> at
>> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterat
>> or(JavaBinUpdateRequestCodec.java:135)
>> at
>> org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:241)
>> at
>> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedL
>> ist(JavaBinUpdateRequestCodec.java:121)
>> at
>> org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:206)
>> at
>> org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:126)
>> at
>> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(Ja
>> vaBinUpdateRequestCodec.java:186)
>> at
>> org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader
>> .java:111)
>> at
>> org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:58)
>> at
>> org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.ja
>> va:98)
>> at
>> org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentS
>> treamHandlerBase.java:74)
>> at
>> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase
>> .java:143)
>> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2064)
>> at
>> org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:654)
>> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:450)
>> at
>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
>> :227)
>> at
>> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
>> :196)
>> at
>> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandle
>> r.java:1652)
>> at
>> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
>> at
>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
>> 3)
>> at
>> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
>> at
>> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.jav
>> a:223)
>> at
>> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.jav
>> a:1127)
>> at
>> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
>> at
>> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java
>> :185)
>> at
>> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java
>> :1061)
>> at
>> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
>> 1)
>> at
>> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHan
>> dlerCollection.java:215)
>> at
>> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection
>> .java:110)
>> at
>> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:
>> 97)
>> at org.eclipse.jetty.server.Server.handle(Server.java:497)
>> at
>> org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
>> at
>> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
>> at
>> org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
>> at
>> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java
>> :635)
>> at
>> org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:
>> 555)
>> at java.lang.Thread.run(Thread.java:745)
>> Caused by: org.apache.lucene.store.AlreadyClosedException: this
>> IndexWriter is closed
>> at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:718)
>> at
>> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:732)
>> at
>> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1359)
>> at
>> org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.ja
>> va:239)
>> at
>> org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.jav
>> a:163)
>> ... 44 more
>> Caused by: java.lang.OutOfMemoryError: Java heap space
>>
>>
>>
>> Thanks
>> Raja
>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: OOM on solr cloud 5.2.1, does not trigger oom_solr.sh
Posted by Erick Erickson <er...@gmail.com>.
There have been a couple of threads lately discussing that some OOMs
are not propagated appropriately and thus don't trigger the OOM killer.
Does anyone think this should be a JIRA?
On Fri, Oct 23, 2015 at 7:17 AM, Raja Pothuganti
<RP...@competitrack.com> wrote:
> Hi,
>
> Some times I see OOM happening on replicas,but does not trigger script
> oom_solr.sh which was passed in as
> -XX:OnOutOfMemoryError=/actualLocation/solr/bin/oom_solr.sh 8091.
>
> These OOM happened while DIH importing data from database. Is this known
> issue? is there any quick fix? Sent yesterday day to users group, no
> response yet.
>
> Here are stack traces when OOM happened
>
>
> 1)
> org.apache.solr.common.SolrException; null:java.lang.RuntimeException:
> java.lang.OutOfMemoryError: Java heap space
> at
> org.apache.solr.servlet.HttpSolrCall.sendError(HttpSolrCall.java:593)
> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:465)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
> :227)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
> :196)
> at
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandle
> r.java:1652)
> at
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
> 3)
> at
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
> at
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.jav
> a:223)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.jav
> a:1127)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java
> :185)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java
> :1061)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
> 1)
> at
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHan
> dlerCollection.java:215)
> at
> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection
> .java:110)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:
> 97)
> at org.eclipse.jetty.server.Server.handle(Server.java:497)
> at
> org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
> at
> org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java
> :635)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:
> 555)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
>
>
> 2)
> org.apache.solr.common.SolrException;
> org.apache.solr.common.SolrException: Exception writing document id
> R277453962 to the index; possible analysis error.
> at
> org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.jav
> a:167)
> at
> org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdatePro
> cessorFactory.java:69)
> at
> org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRe
> questProcessor.java:51)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(Dist
> ributedUpdateProcessor.java:955)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(Dist
> ributedUpdateProcessor.java:1110)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(Dist
> ributedUpdateProcessor.java:706)
> at
> org.apache.solr.update.processor.LogUpdateProcessor.processAdd(LogUpdatePro
> cessorFactory.java:104)
> at
> org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:10
> 1)
> at
> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterM
> ostDocIterator(JavaBinUpdateRequestCodec.java:179)
> at
> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterat
> or(JavaBinUpdateRequestCodec.java:135)
> at
> org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:241)
> at
> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedL
> ist(JavaBinUpdateRequestCodec.java:121)
> at
> org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:206)
> at
> org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:126)
> at
> org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(Ja
> vaBinUpdateRequestCodec.java:186)
> at
> org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader
> .java:111)
> at
> org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:58)
> at
> org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.ja
> va:98)
> at
> org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentS
> treamHandlerBase.java:74)
> at
> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase
> .java:143)
> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2064)
> at
> org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:654)
> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:450)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
> :227)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java
> :196)
> at
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandle
> r.java:1652)
> at
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
> 3)
> at
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
> at
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.jav
> a:223)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.jav
> a:1127)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java
> :185)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java
> :1061)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:14
> 1)
> at
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHan
> dlerCollection.java:215)
> at
> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection
> .java:110)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:
> 97)
> at org.eclipse.jetty.server.Server.handle(Server.java:497)
> at
> org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
> at
> org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java
> :635)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:
> 555)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.lucene.store.AlreadyClosedException: this
> IndexWriter is closed
> at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:718)
> at
> org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:732)
> at
> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1359)
> at
> org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.ja
> va:239)
> at
> org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.jav
> a:163)
> ... 44 more
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
>
>
> Thanks
> Raja
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org