You are viewing a plain text version of this content. The canonical link for it is here.

Posted to solr-user@lucene.apache.org by Modassar Ather <mo...@gmail.com> on 2014/12/23 12:34:40 UTC

Solr server becomes non-responsive.

Hi,

I have a setup of 4 shard Solr cluster with embedded zookeeper on one of
them. The zkClient time out is set to 30 seconds, -Xms is 20g and -Xms is
24g.
When executing a huge query with many wildcards inside it the server
crashes and becomes non-responsive. Even the dashboard does not responds
and shows connection lost error. This requires me to restart the servers.

I have set query *timeAllowed* to 5 minutes but it also seems to be not
getting honored and the query hangs around.

Kindly help me debug the issue and fix it or if there is a way the
timeAllowed can be made honored or a the query which is hanging for
sometime can be stopped.

*Following are few exceptions.*

*org.apache.zookeeper.server.NIOServerCnxn doIO*
WARNING: caught end of stream exception
EndOfStreamException: Unable to read additional data from client sessionid
<session id>, likely client has closed socket
        at
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
        at
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
        at java.lang.Thread.run(Thread.java:745)

*org.apache.zookeeper.server.NIOServerCnxn sendBuffer*
SEVERE: Unexpected Exception:
java.nio.channels.CancelledKeyException
        at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
        at sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77)
        at
org.apache.zookeeper.server.NIOServerCnxn.sendBuffer(NIOServerCnxn.java:151)
        at
org.apache.zookeeper.server.NIOServerCnxn.sendResponse(NIOServerCnxn.java:1081)
        at
org.apache.zookeeper.server.FinalRequestProcessor.processRequest(FinalRequestProcessor.java:404)
        at
org.apache.zookeeper.server.SyncRequestProcessor.flush(SyncRequestProcessor.java:200)
        at
org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:131)

*org.apache.zookeeper.server.persistence.FileTxnLog commit*
WARNING: fsync-ing the write ahead log in SyncThread:0 took 28346ms which
will adversely effect operation latency. See the ZooKeeper troubleshooting
guide
org.apache.solr.common.SolrException log
SEVERE: org.apache.solr.common.SolrException: no servers hosting shard:
        at
org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:149)
        at
org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:119)
        at java.util.concurrent.FutureTask.run(FutureTask.java:262)
        at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
        at java.util.concurrent.FutureTask.run(FutureTask.java:262)
        at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)

*Caused by: java.lang.OutOfMemoryError: Java heap space*
        at
org.apache.lucene.codecs.lucene41.Lucene41PostingsReader$BlockDocsAndPositionsEnum.<init>(Lucene41PostingsReader.java:640)
        at
org.apache.lucene.codecs.lucene41.Lucene41PostingsReader.docsAndPositions(Lucene41PostingsReader.java:278)
        at
org.apache.lucene.codecs.blocktree.SegmentTermsEnum.docsAndPositions(SegmentTermsEnum.java:1011)
        at
org.apache.lucene.search.spans.SpanTermQuery.getSpans(SpanTermQuery.java:123)
        at
org.apache.lucene.search.spans.SpanOrQuery$1.initSpanQueue(SpanOrQuery.java:180)
        at
org.apache.lucene.search.spans.SpanOrQuery$1.next(SpanOrQuery.java:193)
        at
org.apache.lucene.search.spans.SpanOrQuery$1.initSpanQueue(SpanOrQuery.java:182)
        at
org.apache.lucene.search.spans.SpanOrQuery$1.next(SpanOrQuery.java:193)
        at
org.apache.lucene.search.spans.NearSpansUnordered$SpansCell.next(NearSpansUnordered.java:88)
        at
org.apache.lucene.search.spans.NearSpansUnordered.initList(NearSpansUnordered.java:295)
        at
org.apache.lucene.search.spans.NearSpansUnordered.next(NearSpansUnordered.java:164)
        at
org.apache.lucene.search.spans.SpanScorer.<init>(SpanScorer.java:46)
        at
org.apache.lucene.search.spans.SpanWeight.scorer(SpanWeight.java:88)
        at
org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:160)
        at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
        at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
        at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
        at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
        at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
        at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
        at
org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
        at
org.apache.lucene.queries.function.BoostedQuery$BoostedWeight.scorer(BoostedQuery.java:101)
        at org.apache.lucene.search.Weight.bulkScorer(Weight.java:131)
        at
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
        at
org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:297)
        at
org.apache.solr.search.grouping.CommandHandler.searchWithTimeLimiter(CommandHandler.java:219)
        at
org.apache.solr.search.grouping.CommandHandler.execute(CommandHandler.java:154)
        at
org.apache.solr.handler.component.QueryComponent.process(QueryComponent.java:362)
        at
org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:218)
        at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:1967)
        at
org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:777)

Thanks,
Modassar

Re: Solr server becomes non-responsive.

Posted by Shawn Heisey <ap...@elyograg.org>.

On 12/30/2014 4:16 AM, Modassar Ather wrote:
> In the query having lots of wildcard can we put a limitation on number of
> expansion of terms done against a wildcard token something like
> maxBooleanClauses?

I'm not aware of anything for limiting wildcard terms, but I'm willing
to be surprised.

As you've already been told, using ngrams in the index analyzer instead
of wildcards in the query is usually much faster, because it moves the
hard work Solr must do from query time to index time.  It will slow down
"normal" queries (by an unknown amount, it may not be significant)
simply because the index will be larger.  If users continue to use
wildcards after implementing ngrams, those will be even slower than they
are now, because there will be more terms in the index.

Thanks,
Shawn

Re: Solr server becomes non-responsive.

Posted by Modassar Ather <mo...@gmail.com>.

Thanks for you inputs.

Regards,
Modassar

On Tue, Dec 30, 2014 at 8:41 PM, Jack Krupansky <ja...@gmail.com>
wrote:

> I actually did that once as a test years ago, as well as support for
> "paging" through the wildcard terms with a starting offset, and it worked
> great.
>
> One way to think of the feature is as the ability to "sample" the values of
> the wildcard. I mean, not all queries require absolute precision. Sometimes
> you just want to know whether "something" exists matching the pattern, or
> "generally" what the values look like.
>
> I think it would be worth a Jira.
>
>
> -- Jack Krupansky
>
> On Tue, Dec 30, 2014 at 6:16 AM, Modassar Ather <mo...@gmail.com>
> wrote:
>
> > Hi,
> >
> > In the query having lots of wildcard can we put a limitation on number of
> > expansion of terms done against a wildcard token something like
> > maxBooleanClauses?
> >
> > Thanks,
> > Modassar
> >
> > On Mon, Dec 29, 2014 at 11:15 AM, Modassar Ather <modather1981@gmail.com
> >
> > wrote:
> >
> > > Thanks Jack for your suggestions.
> > >
> > > Regards,
> > > Modassar
> > >
> > > On Fri, Dec 26, 2014 at 6:04 PM, Jack Krupansky <
> > jack.krupansky@gmail.com>
> > > wrote:
> > >
> > >> Either you have too little RAM on each node or too much data on each
> > node.
> > >>
> > >> You may need to shard the data much more heavily so that the total
> work
> > on
> > >> a single query is distributed in parallel to more nodes, each node
> > having
> > >> a
> > >> much smaller amount of data to work on.
> > >>
> > >> First, always make sure that the entire Lucene index for each node
> fits
> > >> entirely in the system memory available for file system caching.
> > Otherwise
> > >> the queries will be I/O bound. Check your current queries to see if
> that
> > >> is
> > >> the case - are the nodes compute bound or I/O bound? If I/O bound, add
> > >> more
> > >> system memory until the queries are no longer I/O bound. If compute
> > bound,
> > >> shard more heavily until the query latency becomes acceptable.
> > >>
> > >>
> > >>
> > >> -- Jack Krupansky
> > >>
> > >> On Fri, Dec 26, 2014 at 1:02 AM, Modassar Ather <
> modather1981@gmail.com
> > >
> > >> wrote:
> > >>
> > >> > Thanks for your suggestions Erick.
> > >> >
> > >> > This may be one of those situations where you really have to
> > >> > push back at the users and understand why they insist on these
> > >> > kinds of queries. They must be very patient since it won't be
> > >> > very performant. That said, I've seen this pattern; there are
> > >> > certainly valid conditions under which response times can be
> > >> > many seconds if there are few users and they are doing very
> > >> > complex/expert-level things.
> > >> >
> > >> > We have tried educating the users but it did not work because they
> are
> > >> used
> > >> > to the old way. They feel that wildcard gives more control over the
> > >> results
> > >> > and may not fully understand stemming.
> > >> >
> > >> > Regards,
> > >> > Modassar
> > >> >
> > >> > On Thu, Dec 25, 2014 at 3:17 AM, Erick Erickson <
> > >> erickerickson@gmail.com>
> > >> > wrote:
> > >> >
> > >> > > There's no magic bullet here that I know of. If your requirements
> > >> > > are to support these huge, many-wildcard queries then you only
> > >> > > have a few choices:
> > >> > >
> > >> > > 1> redo the index. I was surprised at how little it bloated the
> > >> > > index as far as memory required is concerned to add ngrams.
> > >> > > The key here is that there really aren't very many unique terms.
> > >> > > If you use bigrams, then there are only maybe 36^2 distinct
> > >> > > combinations. (assuming English and including numbers).
> > >> > >
> > >> > > 2> Increase the number of shards, putting many fewer docs
> > >> > > on each shard.
> > >> > >
> > >> > > 3> give each shard a lot more memory. This isn't actually one
> > >> > > of my preferred solutions since GC issues may raise their ugly
> > >> > > heads here.
> > >> > >
> > >> > > 4> insert creative solution here.
> > >> > >
> > >> > > This may be one of those situations where you really have to
> > >> > > push back at the users and understand why they insist on these
> > >> > > kinds of queries. They must be very patient since it won't be
> > >> > > very performant. That said, I've seen this pattern; there are
> > >> > > certainly valid conditions under which response times can be
> > >> > > many seconds if there are few users and they are doing very
> > >> > > complex/expert-level things.
> > >> > >
> > >> > > Now, all that said, wildcards are often examples of poor habits
> > >> > > or habits learned in DB systems where the only hammer was
> > >> > > %whatever%. I've seen situations where users didn't
> > >> > > understand that Solr broke the input stream up into words. And
> > >> > > stemmed. And WordDelimiterFilterFactory did all the magic
> > >> > > for finding, say D.C. and DC. So it's worth looking at the actual
> > >> > > queries that are sent, perhaps talking to users and understanding
> > >> > > what they _want_ out of the system, then perhaps educating them
> > >> > > as to better ways to get what they want.
> > >> > >
> > >> > > Literally I've seen people insist on entering queries that
> > >> > > wildcarded _everything_ both pre and post wildcards because
> > >> > > they didn't realize that Solr tokenizes...
> > >> > >
> > >> > > Once you hit an OOM, all bets are off as Shawn outlined.
> > >> > >
> > >> > > Best,
> > >> > > Erick
> > >> > >
> > >> > > On Wed, Dec 24, 2014 at 1:57 AM, Modassar Ather <
> > >> modather1981@gmail.com>
> > >> > > wrote:
> > >> > > > Thanks for your response.
> > >> > > >
> > >> > > > How many items in the collection ?
> > >> > > > There are about 100 millions documents.
> > >> > > >
> > >> > > > How are configured cache in solrconfig.xml ?
> > >> > > > Each cache has size attribute as 128.
> > >> > > >
> > >> > > > Can you provide a sample of the query ?
> > >> > > > Does it fail immediately after solrcloud startup or after
> several
> > >> > hours ?
> > >> > > > It is a query with many terms(more than a thousand) and phrase
> > where
> > >> > > > phrases have many wildcards in it.
> > >> > > > Once such query is executed there are many zookeeper related
> > >> exceptions
> > >> > > and
> > >> > > > with a couple of such queries executed it goes for OutOfMemory.
> > >> > > >
> > >> > > > Thanks,
> > >> > > > Modassar
> > >> > > >
> > >> > > >
> > >> > > > On Wed, Dec 24, 2014 at 1:37 PM, Dominique Bejean <
> > >> > > dominique.bejean@eolya.fr
> > >> > > >> wrote:
> > >> > > >
> > >> > > >> And you didn’t give how many RAM on each servers ?
> > >> > > >>
> > >> > > >> 2014-12-24 8:17 GMT+01:00 Dominique Bejean <
> > >> dominique.bejean@eolya.fr
> > >> > >:
> > >> > > >>
> > >> > > >> > Modassar,
> > >> > > >> >
> > >> > > >> > How many items in the collection ?
> > >> > > >> > I mean how many documents per collection ? 1 million, 10
> > >> millions,
> > >> > …?
> > >> > > >> >
> > >> > > >> > How are configured cache in solrconfig.xml ?
> > >> > > >> > What are the size attribute value for each cache ?
> > >> > > >> >
> > >> > > >> > Can you provide a sample of the query ?
> > >> > > >> > Does it fail immediately after solrcloud startup or after
> > several
> > >> > > hours ?
> > >> > > >> >
> > >> > > >> > Dominique
> > >> > > >> >
> > >> > > >> > 2014-12-24 6:20 GMT+01:00 Modassar Ather <
> > modather1981@gmail.com
> > >> >:
> > >> > > >> >
> > >> > > >> >> Thanks for your suggestions.
> > >> > > >> >>
> > >> > > >> >> I will look into the link provided.
> > >> > > >> >>
> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > >> > > >> >>
> > >> > > >> >> This is usually an anti-pattern. The very first thing
> > >> > > >> >> I'd be doing is trying to not do this. See ngrams for infix
> > >> > > >> >> queries, or shingles or ReverseWildcardFilterFactory or.....
> > >> > > >> >>
> > >> > > >> >> We cannot avoid multiple wildcards since that's is our
> user's
> > >> > > >> requirement.
> > >> > > >> >> We try to discourage it but the users insist on firing such
> > >> > queries.
> > >> > > >> Also,
> > >> > > >> >> ngrams etc. can be tried but our index is already huge and
> > >> ngrams
> > >> > may
> > >> > > >> >> further add lot to it. We are OK with such queries failing
> as
> > >> long
> > >> > as
> > >> > > >> >> other
> > >> > > >> >> queries are not affected.
> > >> > > >> >>
> > >> > > >> >>
> > >> > > >> >> Please find the details below.
> > >> > > >> >>
> > >> > > >> >> So, how many nodes in the cluster ?
> > >> > > >> >> There are total 4 nodes on the cluster.
> > >> > > >> >>
> > >> > > >> >> How many shards and replicas for the collection ?
> > >> > > >> >> There are 4 shards and no replica for any of them.
> > >> > > >> >>
> > >> > > >> >> How many items in the collection ?
> > >> > > >> >> If I understand the question correctly there are two
> > collection
> > >> on
> > >> > > each
> > >> > > >> >> node and there size on each node is approximately 190GB and
> > >> 130GB.
> > >> > > >> >>
> > >> > > >> >> What is the size of the index ?
> > >> > > >> >> There are two collection on each node and there size on each
> > >> node
> > >> > is
> > >> > > >> >> approximately 190GB and 130GB.
> > >> > > >> >>
> > >> > > >> >> How is updated the collection (frequency, how many items per
> > >> days,
> > >> > > what
> > >> > > >> is
> > >> > > >> >> your hard commit strategy) ?
> > >> > > >> >> It is an optimized index and read-only. There are no
> > >> inter-mediate
> > >> > > >> update.
> > >> > > >> >>
> > >> > > >> >> How are configured cache in solrconfig.xml ?
> > >> > > >> >> Filter cache, query result cache and document cache are
> > enabled.
> > >> > > >> >> Auto-warming is also done.
> > >> > > >> >>
> > >> > > >> >> Can you provide all other JVM parameters ?
> > >> > > >> >> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
> > >> > > >> >>
> > >> > > >> >> Thanks again,
> > >> > > >> >> Modassar
> > >> > > >> >>
> > >> > > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> > >> > > >> >> dominique.bejean@eolya.fr
> > >> > > >> >> > wrote:
> > >> > > >> >>
> > >> > > >> >> > Hi,
> > >> > > >> >> >
> > >> > > >> >> > I agree Erick it could be a good think to have more
> details
> > >> about
> > >> > > your
> > >> > > >> >> > configuration and collection.
> > >> > > >> >> >
> > >> > > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> > >> > > >> >> >
> > >> > > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr
> servers
> > >> or a
> > >> > > >> >> > collection with 4 shards ?
> > >> > > >> >> >
> > >> > > >> >> > So, how many nodes in the cluster ?
> > >> > > >> >> > How many shards and replicas for the collection ?
> > >> > > >> >> > How many items in the collection ?
> > >> > > >> >> > What is the size of the index ?
> > >> > > >> >> > How is updated the collection (frequency, how many items
> per
> > >> > days,
> > >> > > >> what
> > >> > > >> >> is
> > >> > > >> >> > your hard commit strategy) ?
> > >> > > >> >> > How are configured cache in solrconfig.xml ?
> > >> > > >> >> > Can you provide all other JVM parameters ?
> > >> > > >> >> >
> > >> > > >> >> > Regards
> > >> > > >> >> >
> > >> > > >> >> > Dominique
> > >> > > >> >> >
> > >> > > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
> > >> > erickerickson@gmail.com
> > >> > > >:
> > >> > > >> >> >
> > >> > > >> >> > > Second most important part of your message:
> > >> > > >> >> > > "When executing a huge query with many wildcards inside
> it
> > >> the
> > >> > > >> server"
> > >> > > >> >> > >
> > >> > > >> >> > > This is usually an anti-pattern. The very first thing
> > >> > > >> >> > > I'd be doing is trying to not do this. See ngrams for
> > infix
> > >> > > >> >> > > queries, or shingles or ReverseWildcardFilterFactory
> > or.....
> > >> > > >> >> > >
> > >> > > >> >> > > And if your corpus is very large with many unique terms
> > it's
> > >> > even
> > >> > > >> >> > > worse, but you haven't really told us about that yet.
> > >> > > >> >> > >
> > >> > > >> >> > > Best,
> > >> > > >> >> > > Erick
> > >> > > >> >> > >
> > >> > > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> > >> > > apache@elyograg.org>
> > >> > > >> >> > wrote:
> > >> > > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > >> > > >> >> > > >> Hi,
> > >> > > >> >> > > >>
> > >> > > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> > >> > > zookeeper on
> > >> > > >> >> one
> > >> > > >> >> > of
> > >> > > >> >> > > >> them. The zkClient time out is set to 30 seconds,
> -Xms
> > is
> > >> > 20g
> > >> > > and
> > >> > > >> >> -Xms
> > >> > > >> >> > > is
> > >> > > >> >> > > >> 24g.
> > >> > > >> >> > > >> When executing a huge query with many wildcards
> inside
> > it
> > >> > the
> > >> > > >> >> server
> > >> > > >> >> > > >> crashes and becomes non-responsive. Even the
> dashboard
> > >> does
> > >> > > not
> > >> > > >> >> > responds
> > >> > > >> >> > > >> and shows connection lost error. This requires me to
> > >> restart
> > >> > > the
> > >> > > >> >> > > servers.
> > >> > > >> >> > > >
> > >> > > >> >> > > > Here's the important part of your message:
> > >> > > >> >> > > >
> > >> > > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap
> space*
> > >> > > >> >> > > >
> > >> > > >> >> > > >
> > >> > > >> >> > > > Your heap is not big enough for what Solr has been
> asked
> > >> to
> > >> > do.
> > >> > > >> You
> > >> > > >> >> > > > need to either increase your heap size or change your
> > >> > > >> configuration
> > >> > > >> >> so
> > >> > > >> >> > > > that it uses less memory.
> > >> > > >> >> > > >
> > >> > > >> >> > > >
> > >> > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > >> > > >> >> > > >
> > >> > > >> >> > > > Most programs have pretty much undefined behavior when
> > an
> > >> > OOME
> > >> > > >> >> occurs.
> > >> > > >> >> > > > Lucene's IndexWriter has been hardened so that it
> tries
> > >> > > extremely
> > >> > > >> >> hard
> > >> > > >> >> > > > to avoid index corruption when OOME strikes, and I
> > believe
> > >> > that
> > >> > > >> >> works
> > >> > > >> >> > > > well enough that we can call it nearly bulletproof ...
> > but
> > >> > the
> > >> > > >> rest
> > >> > > >> >> of
> > >> > > >> >> > > > Lucene and Solr will make no guarantees.
> > >> > > >> >> > > >
> > >> > > >> >> > > > It's very difficult to have definable program behavior
> > >> when
> > >> > an
> > >> > > >> OOME
> > >> > > >> >> > > > happens, because you simply cannot know the precise
> > point
> > >> > > during
> > >> > > >> >> > program
> > >> > > >> >> > > > execution where it will happen, or what isn't going to
> > >> work
> > >> > > >> because
> > >> > > >> >> > Java
> > >> > > >> >> > > > did not have memory space to create an object.  Going
> > >> > > unresponsive
> > >> > > >> >> is
> > >> > > >> >> > > > not surprising.
> > >> > > >> >> > > >
> > >> > > >> >> > > > If you can solve your heap problem, note that you may
> > run
> > >> > into
> > >> > > >> other
> > >> > > >> >> > > > performance issues discussed on the wiki page that I
> > >> linked.
> > >> > > >> >> > > >
> > >> > > >> >> > > > Thanks,
> > >> > > >> >> > > > Shawn
> > >> > > >> >> > > >
> > >> > > >> >> > >
> > >> > > >> >> >
> > >> > > >> >>
> > >> > > >> >>
> > >> > > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> > >> > > >> >> dominique.bejean@eolya.fr
> > >> > > >> >> > wrote:
> > >> > > >> >>
> > >> > > >> >> > Hi,
> > >> > > >> >> >
> > >> > > >> >> > I agree Erick it could be a good think to have more
> details
> > >> about
> > >> > > your
> > >> > > >> >> > configuration and collection.
> > >> > > >> >> >
> > >> > > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> > >> > > >> >> >
> > >> > > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr
> servers
> > >> or a
> > >> > > >> >> > collection with 4 shards ?
> > >> > > >> >> >
> > >> > > >> >> > So, how many nodes in the cluster ?
> > >> > > >> >> > How many shards and replicas for the collection ?
> > >> > > >> >> > How many items in the collection ?
> > >> > > >> >> > What is the size of the index ?
> > >> > > >> >> > How is updated the collection (frequency, how many items
> per
> > >> > days,
> > >> > > >> what
> > >> > > >> >> is
> > >> > > >> >> > your hard commit strategy) ?
> > >> > > >> >> > How are configured cache in solrconfig.xml ?
> > >> > > >> >> > Can you provide all other JVM parameters ?
> > >> > > >> >> >
> > >> > > >> >> > Regards
> > >> > > >> >> >
> > >> > > >> >> > Dominique
> > >> > > >> >> >
> > >> > > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
> > >> > erickerickson@gmail.com
> > >> > > >:
> > >> > > >> >> >
> > >> > > >> >> > > Second most important part of your message:
> > >> > > >> >> > > "When executing a huge query with many wildcards inside
> it
> > >> the
> > >> > > >> server"
> > >> > > >> >> > >
> > >> > > >> >> > > This is usually an anti-pattern. The very first thing
> > >> > > >> >> > > I'd be doing is trying to not do this. See ngrams for
> > infix
> > >> > > >> >> > > queries, or shingles or ReverseWildcardFilterFactory
> > or.....
> > >> > > >> >> > >
> > >> > > >> >> > > And if your corpus is very large with many unique terms
> > it's
> > >> > even
> > >> > > >> >> > > worse, but you haven't really told us about that yet.
> > >> > > >> >> > >
> > >> > > >> >> > > Best,
> > >> > > >> >> > > Erick
> > >> > > >> >> > >
> > >> > > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> > >> > > apache@elyograg.org>
> > >> > > >> >> > wrote:
> > >> > > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > >> > > >> >> > > >> Hi,
> > >> > > >> >> > > >>
> > >> > > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> > >> > > zookeeper on
> > >> > > >> >> one
> > >> > > >> >> > of
> > >> > > >> >> > > >> them. The zkClient time out is set to 30 seconds,
> -Xms
> > is
> > >> > 20g
> > >> > > and
> > >> > > >> >> -Xms
> > >> > > >> >> > > is
> > >> > > >> >> > > >> 24g.
> > >> > > >> >> > > >> When executing a huge query with many wildcards
> inside
> > it
> > >> > the
> > >> > > >> >> server
> > >> > > >> >> > > >> crashes and becomes non-responsive. Even the
> dashboard
> > >> does
> > >> > > not
> > >> > > >> >> > responds
> > >> > > >> >> > > >> and shows connection lost error. This requires me to
> > >> restart
> > >> > > the
> > >> > > >> >> > > servers.
> > >> > > >> >> > > >
> > >> > > >> >> > > > Here's the important part of your message:
> > >> > > >> >> > > >
> > >> > > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap
> space*
> > >> > > >> >> > > >
> > >> > > >> >> > > >
> > >> > > >> >> > > > Your heap is not big enough for what Solr has been
> asked
> > >> to
> > >> > do.
> > >> > > >> You
> > >> > > >> >> > > > need to either increase your heap size or change your
> > >> > > >> configuration
> > >> > > >> >> so
> > >> > > >> >> > > > that it uses less memory.
> > >> > > >> >> > > >
> > >> > > >> >> > > >
> > >> > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > >> > > >> >> > > >
> > >> > > >> >> > > > Most programs have pretty much undefined behavior when
> > an
> > >> > OOME
> > >> > > >> >> occurs.
> > >> > > >> >> > > > Lucene's IndexWriter has been hardened so that it
> tries
> > >> > > extremely
> > >> > > >> >> hard
> > >> > > >> >> > > > to avoid index corruption when OOME strikes, and I
> > believe
> > >> > that
> > >> > > >> >> works
> > >> > > >> >> > > > well enough that we can call it nearly bulletproof ...
> > but
> > >> > the
> > >> > > >> rest
> > >> > > >> >> of
> > >> > > >> >> > > > Lucene and Solr will make no guarantees.
> > >> > > >> >> > > >
> > >> > > >> >> > > > It's very difficult to have definable program behavior
> > >> when
> > >> > an
> > >> > > >> OOME
> > >> > > >> >> > > > happens, because you simply cannot know the precise
> > point
> > >> > > during
> > >> > > >> >> > program
> > >> > > >> >> > > > execution where it will happen, or what isn't going to
> > >> work
> > >> > > >> because
> > >> > > >> >> > Java
> > >> > > >> >> > > > did not have memory space to create an object.  Going
> > >> > > unresponsive
> > >> > > >> >> is
> > >> > > >> >> > > > not surprising.
> > >> > > >> >> > > >
> > >> > > >> >> > > > If you can solve your heap problem, note that you may
> > run
> > >> > into
> > >> > > >> other
> > >> > > >> >> > > > performance issues discussed on the wiki page that I
> > >> linked.
> > >> > > >> >> > > >
> > >> > > >> >> > > > Thanks,
> > >> > > >> >> > > > Shawn
> > >> > > >> >> > > >
> > >> > > >> >> > >
> > >> > > >> >> >
> > >> > > >> >>
> > >> > > >> >
> > >> > > >> >
> > >> > > >>
> > >> > >
> > >> >
> > >>
> > >
> > >
> >
>

Re: Solr server becomes non-responsive.

Posted by Jack Krupansky <ja...@gmail.com>.

I actually did that once as a test years ago, as well as support for
"paging" through the wildcard terms with a starting offset, and it worked
great.

One way to think of the feature is as the ability to "sample" the values of
the wildcard. I mean, not all queries require absolute precision. Sometimes
you just want to know whether "something" exists matching the pattern, or
"generally" what the values look like.

I think it would be worth a Jira.


-- Jack Krupansky

On Tue, Dec 30, 2014 at 6:16 AM, Modassar Ather <mo...@gmail.com>
wrote:

> Hi,
>
> In the query having lots of wildcard can we put a limitation on number of
> expansion of terms done against a wildcard token something like
> maxBooleanClauses?
>
> Thanks,
> Modassar
>
> On Mon, Dec 29, 2014 at 11:15 AM, Modassar Ather <mo...@gmail.com>
> wrote:
>
> > Thanks Jack for your suggestions.
> >
> > Regards,
> > Modassar
> >
> > On Fri, Dec 26, 2014 at 6:04 PM, Jack Krupansky <
> jack.krupansky@gmail.com>
> > wrote:
> >
> >> Either you have too little RAM on each node or too much data on each
> node.
> >>
> >> You may need to shard the data much more heavily so that the total work
> on
> >> a single query is distributed in parallel to more nodes, each node
> having
> >> a
> >> much smaller amount of data to work on.
> >>
> >> First, always make sure that the entire Lucene index for each node fits
> >> entirely in the system memory available for file system caching.
> Otherwise
> >> the queries will be I/O bound. Check your current queries to see if that
> >> is
> >> the case - are the nodes compute bound or I/O bound? If I/O bound, add
> >> more
> >> system memory until the queries are no longer I/O bound. If compute
> bound,
> >> shard more heavily until the query latency becomes acceptable.
> >>
> >>
> >>
> >> -- Jack Krupansky
> >>
> >> On Fri, Dec 26, 2014 at 1:02 AM, Modassar Ather <modather1981@gmail.com
> >
> >> wrote:
> >>
> >> > Thanks for your suggestions Erick.
> >> >
> >> > This may be one of those situations where you really have to
> >> > push back at the users and understand why they insist on these
> >> > kinds of queries. They must be very patient since it won't be
> >> > very performant. That said, I've seen this pattern; there are
> >> > certainly valid conditions under which response times can be
> >> > many seconds if there are few users and they are doing very
> >> > complex/expert-level things.
> >> >
> >> > We have tried educating the users but it did not work because they are
> >> used
> >> > to the old way. They feel that wildcard gives more control over the
> >> results
> >> > and may not fully understand stemming.
> >> >
> >> > Regards,
> >> > Modassar
> >> >
> >> > On Thu, Dec 25, 2014 at 3:17 AM, Erick Erickson <
> >> erickerickson@gmail.com>
> >> > wrote:
> >> >
> >> > > There's no magic bullet here that I know of. If your requirements
> >> > > are to support these huge, many-wildcard queries then you only
> >> > > have a few choices:
> >> > >
> >> > > 1> redo the index. I was surprised at how little it bloated the
> >> > > index as far as memory required is concerned to add ngrams.
> >> > > The key here is that there really aren't very many unique terms.
> >> > > If you use bigrams, then there are only maybe 36^2 distinct
> >> > > combinations. (assuming English and including numbers).
> >> > >
> >> > > 2> Increase the number of shards, putting many fewer docs
> >> > > on each shard.
> >> > >
> >> > > 3> give each shard a lot more memory. This isn't actually one
> >> > > of my preferred solutions since GC issues may raise their ugly
> >> > > heads here.
> >> > >
> >> > > 4> insert creative solution here.
> >> > >
> >> > > This may be one of those situations where you really have to
> >> > > push back at the users and understand why they insist on these
> >> > > kinds of queries. They must be very patient since it won't be
> >> > > very performant. That said, I've seen this pattern; there are
> >> > > certainly valid conditions under which response times can be
> >> > > many seconds if there are few users and they are doing very
> >> > > complex/expert-level things.
> >> > >
> >> > > Now, all that said, wildcards are often examples of poor habits
> >> > > or habits learned in DB systems where the only hammer was
> >> > > %whatever%. I've seen situations where users didn't
> >> > > understand that Solr broke the input stream up into words. And
> >> > > stemmed. And WordDelimiterFilterFactory did all the magic
> >> > > for finding, say D.C. and DC. So it's worth looking at the actual
> >> > > queries that are sent, perhaps talking to users and understanding
> >> > > what they _want_ out of the system, then perhaps educating them
> >> > > as to better ways to get what they want.
> >> > >
> >> > > Literally I've seen people insist on entering queries that
> >> > > wildcarded _everything_ both pre and post wildcards because
> >> > > they didn't realize that Solr tokenizes...
> >> > >
> >> > > Once you hit an OOM, all bets are off as Shawn outlined.
> >> > >
> >> > > Best,
> >> > > Erick
> >> > >
> >> > > On Wed, Dec 24, 2014 at 1:57 AM, Modassar Ather <
> >> modather1981@gmail.com>
> >> > > wrote:
> >> > > > Thanks for your response.
> >> > > >
> >> > > > How many items in the collection ?
> >> > > > There are about 100 millions documents.
> >> > > >
> >> > > > How are configured cache in solrconfig.xml ?
> >> > > > Each cache has size attribute as 128.
> >> > > >
> >> > > > Can you provide a sample of the query ?
> >> > > > Does it fail immediately after solrcloud startup or after several
> >> > hours ?
> >> > > > It is a query with many terms(more than a thousand) and phrase
> where
> >> > > > phrases have many wildcards in it.
> >> > > > Once such query is executed there are many zookeeper related
> >> exceptions
> >> > > and
> >> > > > with a couple of such queries executed it goes for OutOfMemory.
> >> > > >
> >> > > > Thanks,
> >> > > > Modassar
> >> > > >
> >> > > >
> >> > > > On Wed, Dec 24, 2014 at 1:37 PM, Dominique Bejean <
> >> > > dominique.bejean@eolya.fr
> >> > > >> wrote:
> >> > > >
> >> > > >> And you didn’t give how many RAM on each servers ?
> >> > > >>
> >> > > >> 2014-12-24 8:17 GMT+01:00 Dominique Bejean <
> >> dominique.bejean@eolya.fr
> >> > >:
> >> > > >>
> >> > > >> > Modassar,
> >> > > >> >
> >> > > >> > How many items in the collection ?
> >> > > >> > I mean how many documents per collection ? 1 million, 10
> >> millions,
> >> > …?
> >> > > >> >
> >> > > >> > How are configured cache in solrconfig.xml ?
> >> > > >> > What are the size attribute value for each cache ?
> >> > > >> >
> >> > > >> > Can you provide a sample of the query ?
> >> > > >> > Does it fail immediately after solrcloud startup or after
> several
> >> > > hours ?
> >> > > >> >
> >> > > >> > Dominique
> >> > > >> >
> >> > > >> > 2014-12-24 6:20 GMT+01:00 Modassar Ather <
> modather1981@gmail.com
> >> >:
> >> > > >> >
> >> > > >> >> Thanks for your suggestions.
> >> > > >> >>
> >> > > >> >> I will look into the link provided.
> >> > > >> >> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >> > > >> >>
> >> > > >> >> This is usually an anti-pattern. The very first thing
> >> > > >> >> I'd be doing is trying to not do this. See ngrams for infix
> >> > > >> >> queries, or shingles or ReverseWildcardFilterFactory or.....
> >> > > >> >>
> >> > > >> >> We cannot avoid multiple wildcards since that's is our user's
> >> > > >> requirement.
> >> > > >> >> We try to discourage it but the users insist on firing such
> >> > queries.
> >> > > >> Also,
> >> > > >> >> ngrams etc. can be tried but our index is already huge and
> >> ngrams
> >> > may
> >> > > >> >> further add lot to it. We are OK with such queries failing as
> >> long
> >> > as
> >> > > >> >> other
> >> > > >> >> queries are not affected.
> >> > > >> >>
> >> > > >> >>
> >> > > >> >> Please find the details below.
> >> > > >> >>
> >> > > >> >> So, how many nodes in the cluster ?
> >> > > >> >> There are total 4 nodes on the cluster.
> >> > > >> >>
> >> > > >> >> How many shards and replicas for the collection ?
> >> > > >> >> There are 4 shards and no replica for any of them.
> >> > > >> >>
> >> > > >> >> How many items in the collection ?
> >> > > >> >> If I understand the question correctly there are two
> collection
> >> on
> >> > > each
> >> > > >> >> node and there size on each node is approximately 190GB and
> >> 130GB.
> >> > > >> >>
> >> > > >> >> What is the size of the index ?
> >> > > >> >> There are two collection on each node and there size on each
> >> node
> >> > is
> >> > > >> >> approximately 190GB and 130GB.
> >> > > >> >>
> >> > > >> >> How is updated the collection (frequency, how many items per
> >> days,
> >> > > what
> >> > > >> is
> >> > > >> >> your hard commit strategy) ?
> >> > > >> >> It is an optimized index and read-only. There are no
> >> inter-mediate
> >> > > >> update.
> >> > > >> >>
> >> > > >> >> How are configured cache in solrconfig.xml ?
> >> > > >> >> Filter cache, query result cache and document cache are
> enabled.
> >> > > >> >> Auto-warming is also done.
> >> > > >> >>
> >> > > >> >> Can you provide all other JVM parameters ?
> >> > > >> >> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
> >> > > >> >>
> >> > > >> >> Thanks again,
> >> > > >> >> Modassar
> >> > > >> >>
> >> > > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> >> > > >> >> dominique.bejean@eolya.fr
> >> > > >> >> > wrote:
> >> > > >> >>
> >> > > >> >> > Hi,
> >> > > >> >> >
> >> > > >> >> > I agree Erick it could be a good think to have more details
> >> about
> >> > > your
> >> > > >> >> > configuration and collection.
> >> > > >> >> >
> >> > > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> >> > > >> >> >
> >> > > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers
> >> or a
> >> > > >> >> > collection with 4 shards ?
> >> > > >> >> >
> >> > > >> >> > So, how many nodes in the cluster ?
> >> > > >> >> > How many shards and replicas for the collection ?
> >> > > >> >> > How many items in the collection ?
> >> > > >> >> > What is the size of the index ?
> >> > > >> >> > How is updated the collection (frequency, how many items per
> >> > days,
> >> > > >> what
> >> > > >> >> is
> >> > > >> >> > your hard commit strategy) ?
> >> > > >> >> > How are configured cache in solrconfig.xml ?
> >> > > >> >> > Can you provide all other JVM parameters ?
> >> > > >> >> >
> >> > > >> >> > Regards
> >> > > >> >> >
> >> > > >> >> > Dominique
> >> > > >> >> >
> >> > > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
> >> > erickerickson@gmail.com
> >> > > >:
> >> > > >> >> >
> >> > > >> >> > > Second most important part of your message:
> >> > > >> >> > > "When executing a huge query with many wildcards inside it
> >> the
> >> > > >> server"
> >> > > >> >> > >
> >> > > >> >> > > This is usually an anti-pattern. The very first thing
> >> > > >> >> > > I'd be doing is trying to not do this. See ngrams for
> infix
> >> > > >> >> > > queries, or shingles or ReverseWildcardFilterFactory
> or.....
> >> > > >> >> > >
> >> > > >> >> > > And if your corpus is very large with many unique terms
> it's
> >> > even
> >> > > >> >> > > worse, but you haven't really told us about that yet.
> >> > > >> >> > >
> >> > > >> >> > > Best,
> >> > > >> >> > > Erick
> >> > > >> >> > >
> >> > > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> >> > > apache@elyograg.org>
> >> > > >> >> > wrote:
> >> > > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> >> > > >> >> > > >> Hi,
> >> > > >> >> > > >>
> >> > > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> >> > > zookeeper on
> >> > > >> >> one
> >> > > >> >> > of
> >> > > >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms
> is
> >> > 20g
> >> > > and
> >> > > >> >> -Xms
> >> > > >> >> > > is
> >> > > >> >> > > >> 24g.
> >> > > >> >> > > >> When executing a huge query with many wildcards inside
> it
> >> > the
> >> > > >> >> server
> >> > > >> >> > > >> crashes and becomes non-responsive. Even the dashboard
> >> does
> >> > > not
> >> > > >> >> > responds
> >> > > >> >> > > >> and shows connection lost error. This requires me to
> >> restart
> >> > > the
> >> > > >> >> > > servers.
> >> > > >> >> > > >
> >> > > >> >> > > > Here's the important part of your message:
> >> > > >> >> > > >
> >> > > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> >> > > >> >> > > >
> >> > > >> >> > > >
> >> > > >> >> > > > Your heap is not big enough for what Solr has been asked
> >> to
> >> > do.
> >> > > >> You
> >> > > >> >> > > > need to either increase your heap size or change your
> >> > > >> configuration
> >> > > >> >> so
> >> > > >> >> > > > that it uses less memory.
> >> > > >> >> > > >
> >> > > >> >> > > >
> >> > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >> > > >> >> > > >
> >> > > >> >> > > > Most programs have pretty much undefined behavior when
> an
> >> > OOME
> >> > > >> >> occurs.
> >> > > >> >> > > > Lucene's IndexWriter has been hardened so that it tries
> >> > > extremely
> >> > > >> >> hard
> >> > > >> >> > > > to avoid index corruption when OOME strikes, and I
> believe
> >> > that
> >> > > >> >> works
> >> > > >> >> > > > well enough that we can call it nearly bulletproof ...
> but
> >> > the
> >> > > >> rest
> >> > > >> >> of
> >> > > >> >> > > > Lucene and Solr will make no guarantees.
> >> > > >> >> > > >
> >> > > >> >> > > > It's very difficult to have definable program behavior
> >> when
> >> > an
> >> > > >> OOME
> >> > > >> >> > > > happens, because you simply cannot know the precise
> point
> >> > > during
> >> > > >> >> > program
> >> > > >> >> > > > execution where it will happen, or what isn't going to
> >> work
> >> > > >> because
> >> > > >> >> > Java
> >> > > >> >> > > > did not have memory space to create an object.  Going
> >> > > unresponsive
> >> > > >> >> is
> >> > > >> >> > > > not surprising.
> >> > > >> >> > > >
> >> > > >> >> > > > If you can solve your heap problem, note that you may
> run
> >> > into
> >> > > >> other
> >> > > >> >> > > > performance issues discussed on the wiki page that I
> >> linked.
> >> > > >> >> > > >
> >> > > >> >> > > > Thanks,
> >> > > >> >> > > > Shawn
> >> > > >> >> > > >
> >> > > >> >> > >
> >> > > >> >> >
> >> > > >> >>
> >> > > >> >>
> >> > > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> >> > > >> >> dominique.bejean@eolya.fr
> >> > > >> >> > wrote:
> >> > > >> >>
> >> > > >> >> > Hi,
> >> > > >> >> >
> >> > > >> >> > I agree Erick it could be a good think to have more details
> >> about
> >> > > your
> >> > > >> >> > configuration and collection.
> >> > > >> >> >
> >> > > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> >> > > >> >> >
> >> > > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers
> >> or a
> >> > > >> >> > collection with 4 shards ?
> >> > > >> >> >
> >> > > >> >> > So, how many nodes in the cluster ?
> >> > > >> >> > How many shards and replicas for the collection ?
> >> > > >> >> > How many items in the collection ?
> >> > > >> >> > What is the size of the index ?
> >> > > >> >> > How is updated the collection (frequency, how many items per
> >> > days,
> >> > > >> what
> >> > > >> >> is
> >> > > >> >> > your hard commit strategy) ?
> >> > > >> >> > How are configured cache in solrconfig.xml ?
> >> > > >> >> > Can you provide all other JVM parameters ?
> >> > > >> >> >
> >> > > >> >> > Regards
> >> > > >> >> >
> >> > > >> >> > Dominique
> >> > > >> >> >
> >> > > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
> >> > erickerickson@gmail.com
> >> > > >:
> >> > > >> >> >
> >> > > >> >> > > Second most important part of your message:
> >> > > >> >> > > "When executing a huge query with many wildcards inside it
> >> the
> >> > > >> server"
> >> > > >> >> > >
> >> > > >> >> > > This is usually an anti-pattern. The very first thing
> >> > > >> >> > > I'd be doing is trying to not do this. See ngrams for
> infix
> >> > > >> >> > > queries, or shingles or ReverseWildcardFilterFactory
> or.....
> >> > > >> >> > >
> >> > > >> >> > > And if your corpus is very large with many unique terms
> it's
> >> > even
> >> > > >> >> > > worse, but you haven't really told us about that yet.
> >> > > >> >> > >
> >> > > >> >> > > Best,
> >> > > >> >> > > Erick
> >> > > >> >> > >
> >> > > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> >> > > apache@elyograg.org>
> >> > > >> >> > wrote:
> >> > > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> >> > > >> >> > > >> Hi,
> >> > > >> >> > > >>
> >> > > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> >> > > zookeeper on
> >> > > >> >> one
> >> > > >> >> > of
> >> > > >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms
> is
> >> > 20g
> >> > > and
> >> > > >> >> -Xms
> >> > > >> >> > > is
> >> > > >> >> > > >> 24g.
> >> > > >> >> > > >> When executing a huge query with many wildcards inside
> it
> >> > the
> >> > > >> >> server
> >> > > >> >> > > >> crashes and becomes non-responsive. Even the dashboard
> >> does
> >> > > not
> >> > > >> >> > responds
> >> > > >> >> > > >> and shows connection lost error. This requires me to
> >> restart
> >> > > the
> >> > > >> >> > > servers.
> >> > > >> >> > > >
> >> > > >> >> > > > Here's the important part of your message:
> >> > > >> >> > > >
> >> > > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> >> > > >> >> > > >
> >> > > >> >> > > >
> >> > > >> >> > > > Your heap is not big enough for what Solr has been asked
> >> to
> >> > do.
> >> > > >> You
> >> > > >> >> > > > need to either increase your heap size or change your
> >> > > >> configuration
> >> > > >> >> so
> >> > > >> >> > > > that it uses less memory.
> >> > > >> >> > > >
> >> > > >> >> > > >
> >> > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >> > > >> >> > > >
> >> > > >> >> > > > Most programs have pretty much undefined behavior when
> an
> >> > OOME
> >> > > >> >> occurs.
> >> > > >> >> > > > Lucene's IndexWriter has been hardened so that it tries
> >> > > extremely
> >> > > >> >> hard
> >> > > >> >> > > > to avoid index corruption when OOME strikes, and I
> believe
> >> > that
> >> > > >> >> works
> >> > > >> >> > > > well enough that we can call it nearly bulletproof ...
> but
> >> > the
> >> > > >> rest
> >> > > >> >> of
> >> > > >> >> > > > Lucene and Solr will make no guarantees.
> >> > > >> >> > > >
> >> > > >> >> > > > It's very difficult to have definable program behavior
> >> when
> >> > an
> >> > > >> OOME
> >> > > >> >> > > > happens, because you simply cannot know the precise
> point
> >> > > during
> >> > > >> >> > program
> >> > > >> >> > > > execution where it will happen, or what isn't going to
> >> work
> >> > > >> because
> >> > > >> >> > Java
> >> > > >> >> > > > did not have memory space to create an object.  Going
> >> > > unresponsive
> >> > > >> >> is
> >> > > >> >> > > > not surprising.
> >> > > >> >> > > >
> >> > > >> >> > > > If you can solve your heap problem, note that you may
> run
> >> > into
> >> > > >> other
> >> > > >> >> > > > performance issues discussed on the wiki page that I
> >> linked.
> >> > > >> >> > > >
> >> > > >> >> > > > Thanks,
> >> > > >> >> > > > Shawn
> >> > > >> >> > > >
> >> > > >> >> > >
> >> > > >> >> >
> >> > > >> >>
> >> > > >> >
> >> > > >> >
> >> > > >>
> >> > >
> >> >
> >>
> >
> >
>

Re: Solr server becomes non-responsive.

Posted by Modassar Ather <mo...@gmail.com>.

Hi,

In the query having lots of wildcard can we put a limitation on number of
expansion of terms done against a wildcard token something like
maxBooleanClauses?

Thanks,
Modassar

On Mon, Dec 29, 2014 at 11:15 AM, Modassar Ather <mo...@gmail.com>
wrote:

> Thanks Jack for your suggestions.
>
> Regards,
> Modassar
>
> On Fri, Dec 26, 2014 at 6:04 PM, Jack Krupansky <ja...@gmail.com>
> wrote:
>
>> Either you have too little RAM on each node or too much data on each node.
>>
>> You may need to shard the data much more heavily so that the total work on
>> a single query is distributed in parallel to more nodes, each node having
>> a
>> much smaller amount of data to work on.
>>
>> First, always make sure that the entire Lucene index for each node fits
>> entirely in the system memory available for file system caching. Otherwise
>> the queries will be I/O bound. Check your current queries to see if that
>> is
>> the case - are the nodes compute bound or I/O bound? If I/O bound, add
>> more
>> system memory until the queries are no longer I/O bound. If compute bound,
>> shard more heavily until the query latency becomes acceptable.
>>
>>
>>
>> -- Jack Krupansky
>>
>> On Fri, Dec 26, 2014 at 1:02 AM, Modassar Ather <mo...@gmail.com>
>> wrote:
>>
>> > Thanks for your suggestions Erick.
>> >
>> > This may be one of those situations where you really have to
>> > push back at the users and understand why they insist on these
>> > kinds of queries. They must be very patient since it won't be
>> > very performant. That said, I've seen this pattern; there are
>> > certainly valid conditions under which response times can be
>> > many seconds if there are few users and they are doing very
>> > complex/expert-level things.
>> >
>> > We have tried educating the users but it did not work because they are
>> used
>> > to the old way. They feel that wildcard gives more control over the
>> results
>> > and may not fully understand stemming.
>> >
>> > Regards,
>> > Modassar
>> >
>> > On Thu, Dec 25, 2014 at 3:17 AM, Erick Erickson <
>> erickerickson@gmail.com>
>> > wrote:
>> >
>> > > There's no magic bullet here that I know of. If your requirements
>> > > are to support these huge, many-wildcard queries then you only
>> > > have a few choices:
>> > >
>> > > 1> redo the index. I was surprised at how little it bloated the
>> > > index as far as memory required is concerned to add ngrams.
>> > > The key here is that there really aren't very many unique terms.
>> > > If you use bigrams, then there are only maybe 36^2 distinct
>> > > combinations. (assuming English and including numbers).
>> > >
>> > > 2> Increase the number of shards, putting many fewer docs
>> > > on each shard.
>> > >
>> > > 3> give each shard a lot more memory. This isn't actually one
>> > > of my preferred solutions since GC issues may raise their ugly
>> > > heads here.
>> > >
>> > > 4> insert creative solution here.
>> > >
>> > > This may be one of those situations where you really have to
>> > > push back at the users and understand why they insist on these
>> > > kinds of queries. They must be very patient since it won't be
>> > > very performant. That said, I've seen this pattern; there are
>> > > certainly valid conditions under which response times can be
>> > > many seconds if there are few users and they are doing very
>> > > complex/expert-level things.
>> > >
>> > > Now, all that said, wildcards are often examples of poor habits
>> > > or habits learned in DB systems where the only hammer was
>> > > %whatever%. I've seen situations where users didn't
>> > > understand that Solr broke the input stream up into words. And
>> > > stemmed. And WordDelimiterFilterFactory did all the magic
>> > > for finding, say D.C. and DC. So it's worth looking at the actual
>> > > queries that are sent, perhaps talking to users and understanding
>> > > what they _want_ out of the system, then perhaps educating them
>> > > as to better ways to get what they want.
>> > >
>> > > Literally I've seen people insist on entering queries that
>> > > wildcarded _everything_ both pre and post wildcards because
>> > > they didn't realize that Solr tokenizes...
>> > >
>> > > Once you hit an OOM, all bets are off as Shawn outlined.
>> > >
>> > > Best,
>> > > Erick
>> > >
>> > > On Wed, Dec 24, 2014 at 1:57 AM, Modassar Ather <
>> modather1981@gmail.com>
>> > > wrote:
>> > > > Thanks for your response.
>> > > >
>> > > > How many items in the collection ?
>> > > > There are about 100 millions documents.
>> > > >
>> > > > How are configured cache in solrconfig.xml ?
>> > > > Each cache has size attribute as 128.
>> > > >
>> > > > Can you provide a sample of the query ?
>> > > > Does it fail immediately after solrcloud startup or after several
>> > hours ?
>> > > > It is a query with many terms(more than a thousand) and phrase where
>> > > > phrases have many wildcards in it.
>> > > > Once such query is executed there are many zookeeper related
>> exceptions
>> > > and
>> > > > with a couple of such queries executed it goes for OutOfMemory.
>> > > >
>> > > > Thanks,
>> > > > Modassar
>> > > >
>> > > >
>> > > > On Wed, Dec 24, 2014 at 1:37 PM, Dominique Bejean <
>> > > dominique.bejean@eolya.fr
>> > > >> wrote:
>> > > >
>> > > >> And you didn’t give how many RAM on each servers ?
>> > > >>
>> > > >> 2014-12-24 8:17 GMT+01:00 Dominique Bejean <
>> dominique.bejean@eolya.fr
>> > >:
>> > > >>
>> > > >> > Modassar,
>> > > >> >
>> > > >> > How many items in the collection ?
>> > > >> > I mean how many documents per collection ? 1 million, 10
>> millions,
>> > …?
>> > > >> >
>> > > >> > How are configured cache in solrconfig.xml ?
>> > > >> > What are the size attribute value for each cache ?
>> > > >> >
>> > > >> > Can you provide a sample of the query ?
>> > > >> > Does it fail immediately after solrcloud startup or after several
>> > > hours ?
>> > > >> >
>> > > >> > Dominique
>> > > >> >
>> > > >> > 2014-12-24 6:20 GMT+01:00 Modassar Ather <modather1981@gmail.com
>> >:
>> > > >> >
>> > > >> >> Thanks for your suggestions.
>> > > >> >>
>> > > >> >> I will look into the link provided.
>> > > >> >> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>> > > >> >>
>> > > >> >> This is usually an anti-pattern. The very first thing
>> > > >> >> I'd be doing is trying to not do this. See ngrams for infix
>> > > >> >> queries, or shingles or ReverseWildcardFilterFactory or.....
>> > > >> >>
>> > > >> >> We cannot avoid multiple wildcards since that's is our user's
>> > > >> requirement.
>> > > >> >> We try to discourage it but the users insist on firing such
>> > queries.
>> > > >> Also,
>> > > >> >> ngrams etc. can be tried but our index is already huge and
>> ngrams
>> > may
>> > > >> >> further add lot to it. We are OK with such queries failing as
>> long
>> > as
>> > > >> >> other
>> > > >> >> queries are not affected.
>> > > >> >>
>> > > >> >>
>> > > >> >> Please find the details below.
>> > > >> >>
>> > > >> >> So, how many nodes in the cluster ?
>> > > >> >> There are total 4 nodes on the cluster.
>> > > >> >>
>> > > >> >> How many shards and replicas for the collection ?
>> > > >> >> There are 4 shards and no replica for any of them.
>> > > >> >>
>> > > >> >> How many items in the collection ?
>> > > >> >> If I understand the question correctly there are two collection
>> on
>> > > each
>> > > >> >> node and there size on each node is approximately 190GB and
>> 130GB.
>> > > >> >>
>> > > >> >> What is the size of the index ?
>> > > >> >> There are two collection on each node and there size on each
>> node
>> > is
>> > > >> >> approximately 190GB and 130GB.
>> > > >> >>
>> > > >> >> How is updated the collection (frequency, how many items per
>> days,
>> > > what
>> > > >> is
>> > > >> >> your hard commit strategy) ?
>> > > >> >> It is an optimized index and read-only. There are no
>> inter-mediate
>> > > >> update.
>> > > >> >>
>> > > >> >> How are configured cache in solrconfig.xml ?
>> > > >> >> Filter cache, query result cache and document cache are enabled.
>> > > >> >> Auto-warming is also done.
>> > > >> >>
>> > > >> >> Can you provide all other JVM parameters ?
>> > > >> >> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
>> > > >> >>
>> > > >> >> Thanks again,
>> > > >> >> Modassar
>> > > >> >>
>> > > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
>> > > >> >> dominique.bejean@eolya.fr
>> > > >> >> > wrote:
>> > > >> >>
>> > > >> >> > Hi,
>> > > >> >> >
>> > > >> >> > I agree Erick it could be a good think to have more details
>> about
>> > > your
>> > > >> >> > configuration and collection.
>> > > >> >> >
>> > > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
>> > > >> >> >
>> > > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers
>> or a
>> > > >> >> > collection with 4 shards ?
>> > > >> >> >
>> > > >> >> > So, how many nodes in the cluster ?
>> > > >> >> > How many shards and replicas for the collection ?
>> > > >> >> > How many items in the collection ?
>> > > >> >> > What is the size of the index ?
>> > > >> >> > How is updated the collection (frequency, how many items per
>> > days,
>> > > >> what
>> > > >> >> is
>> > > >> >> > your hard commit strategy) ?
>> > > >> >> > How are configured cache in solrconfig.xml ?
>> > > >> >> > Can you provide all other JVM parameters ?
>> > > >> >> >
>> > > >> >> > Regards
>> > > >> >> >
>> > > >> >> > Dominique
>> > > >> >> >
>> > > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
>> > erickerickson@gmail.com
>> > > >:
>> > > >> >> >
>> > > >> >> > > Second most important part of your message:
>> > > >> >> > > "When executing a huge query with many wildcards inside it
>> the
>> > > >> server"
>> > > >> >> > >
>> > > >> >> > > This is usually an anti-pattern. The very first thing
>> > > >> >> > > I'd be doing is trying to not do this. See ngrams for infix
>> > > >> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
>> > > >> >> > >
>> > > >> >> > > And if your corpus is very large with many unique terms it's
>> > even
>> > > >> >> > > worse, but you haven't really told us about that yet.
>> > > >> >> > >
>> > > >> >> > > Best,
>> > > >> >> > > Erick
>> > > >> >> > >
>> > > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
>> > > apache@elyograg.org>
>> > > >> >> > wrote:
>> > > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
>> > > >> >> > > >> Hi,
>> > > >> >> > > >>
>> > > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
>> > > zookeeper on
>> > > >> >> one
>> > > >> >> > of
>> > > >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is
>> > 20g
>> > > and
>> > > >> >> -Xms
>> > > >> >> > > is
>> > > >> >> > > >> 24g.
>> > > >> >> > > >> When executing a huge query with many wildcards inside it
>> > the
>> > > >> >> server
>> > > >> >> > > >> crashes and becomes non-responsive. Even the dashboard
>> does
>> > > not
>> > > >> >> > responds
>> > > >> >> > > >> and shows connection lost error. This requires me to
>> restart
>> > > the
>> > > >> >> > > servers.
>> > > >> >> > > >
>> > > >> >> > > > Here's the important part of your message:
>> > > >> >> > > >
>> > > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
>> > > >> >> > > >
>> > > >> >> > > >
>> > > >> >> > > > Your heap is not big enough for what Solr has been asked
>> to
>> > do.
>> > > >> You
>> > > >> >> > > > need to either increase your heap size or change your
>> > > >> configuration
>> > > >> >> so
>> > > >> >> > > > that it uses less memory.
>> > > >> >> > > >
>> > > >> >> > > >
>> > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>> > > >> >> > > >
>> > > >> >> > > > Most programs have pretty much undefined behavior when an
>> > OOME
>> > > >> >> occurs.
>> > > >> >> > > > Lucene's IndexWriter has been hardened so that it tries
>> > > extremely
>> > > >> >> hard
>> > > >> >> > > > to avoid index corruption when OOME strikes, and I believe
>> > that
>> > > >> >> works
>> > > >> >> > > > well enough that we can call it nearly bulletproof ... but
>> > the
>> > > >> rest
>> > > >> >> of
>> > > >> >> > > > Lucene and Solr will make no guarantees.
>> > > >> >> > > >
>> > > >> >> > > > It's very difficult to have definable program behavior
>> when
>> > an
>> > > >> OOME
>> > > >> >> > > > happens, because you simply cannot know the precise point
>> > > during
>> > > >> >> > program
>> > > >> >> > > > execution where it will happen, or what isn't going to
>> work
>> > > >> because
>> > > >> >> > Java
>> > > >> >> > > > did not have memory space to create an object.  Going
>> > > unresponsive
>> > > >> >> is
>> > > >> >> > > > not surprising.
>> > > >> >> > > >
>> > > >> >> > > > If you can solve your heap problem, note that you may run
>> > into
>> > > >> other
>> > > >> >> > > > performance issues discussed on the wiki page that I
>> linked.
>> > > >> >> > > >
>> > > >> >> > > > Thanks,
>> > > >> >> > > > Shawn
>> > > >> >> > > >
>> > > >> >> > >
>> > > >> >> >
>> > > >> >>
>> > > >> >>
>> > > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
>> > > >> >> dominique.bejean@eolya.fr
>> > > >> >> > wrote:
>> > > >> >>
>> > > >> >> > Hi,
>> > > >> >> >
>> > > >> >> > I agree Erick it could be a good think to have more details
>> about
>> > > your
>> > > >> >> > configuration and collection.
>> > > >> >> >
>> > > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
>> > > >> >> >
>> > > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers
>> or a
>> > > >> >> > collection with 4 shards ?
>> > > >> >> >
>> > > >> >> > So, how many nodes in the cluster ?
>> > > >> >> > How many shards and replicas for the collection ?
>> > > >> >> > How many items in the collection ?
>> > > >> >> > What is the size of the index ?
>> > > >> >> > How is updated the collection (frequency, how many items per
>> > days,
>> > > >> what
>> > > >> >> is
>> > > >> >> > your hard commit strategy) ?
>> > > >> >> > How are configured cache in solrconfig.xml ?
>> > > >> >> > Can you provide all other JVM parameters ?
>> > > >> >> >
>> > > >> >> > Regards
>> > > >> >> >
>> > > >> >> > Dominique
>> > > >> >> >
>> > > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
>> > erickerickson@gmail.com
>> > > >:
>> > > >> >> >
>> > > >> >> > > Second most important part of your message:
>> > > >> >> > > "When executing a huge query with many wildcards inside it
>> the
>> > > >> server"
>> > > >> >> > >
>> > > >> >> > > This is usually an anti-pattern. The very first thing
>> > > >> >> > > I'd be doing is trying to not do this. See ngrams for infix
>> > > >> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
>> > > >> >> > >
>> > > >> >> > > And if your corpus is very large with many unique terms it's
>> > even
>> > > >> >> > > worse, but you haven't really told us about that yet.
>> > > >> >> > >
>> > > >> >> > > Best,
>> > > >> >> > > Erick
>> > > >> >> > >
>> > > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
>> > > apache@elyograg.org>
>> > > >> >> > wrote:
>> > > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
>> > > >> >> > > >> Hi,
>> > > >> >> > > >>
>> > > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
>> > > zookeeper on
>> > > >> >> one
>> > > >> >> > of
>> > > >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is
>> > 20g
>> > > and
>> > > >> >> -Xms
>> > > >> >> > > is
>> > > >> >> > > >> 24g.
>> > > >> >> > > >> When executing a huge query with many wildcards inside it
>> > the
>> > > >> >> server
>> > > >> >> > > >> crashes and becomes non-responsive. Even the dashboard
>> does
>> > > not
>> > > >> >> > responds
>> > > >> >> > > >> and shows connection lost error. This requires me to
>> restart
>> > > the
>> > > >> >> > > servers.
>> > > >> >> > > >
>> > > >> >> > > > Here's the important part of your message:
>> > > >> >> > > >
>> > > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
>> > > >> >> > > >
>> > > >> >> > > >
>> > > >> >> > > > Your heap is not big enough for what Solr has been asked
>> to
>> > do.
>> > > >> You
>> > > >> >> > > > need to either increase your heap size or change your
>> > > >> configuration
>> > > >> >> so
>> > > >> >> > > > that it uses less memory.
>> > > >> >> > > >
>> > > >> >> > > >
>> > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>> > > >> >> > > >
>> > > >> >> > > > Most programs have pretty much undefined behavior when an
>> > OOME
>> > > >> >> occurs.
>> > > >> >> > > > Lucene's IndexWriter has been hardened so that it tries
>> > > extremely
>> > > >> >> hard
>> > > >> >> > > > to avoid index corruption when OOME strikes, and I believe
>> > that
>> > > >> >> works
>> > > >> >> > > > well enough that we can call it nearly bulletproof ... but
>> > the
>> > > >> rest
>> > > >> >> of
>> > > >> >> > > > Lucene and Solr will make no guarantees.
>> > > >> >> > > >
>> > > >> >> > > > It's very difficult to have definable program behavior
>> when
>> > an
>> > > >> OOME
>> > > >> >> > > > happens, because you simply cannot know the precise point
>> > > during
>> > > >> >> > program
>> > > >> >> > > > execution where it will happen, or what isn't going to
>> work
>> > > >> because
>> > > >> >> > Java
>> > > >> >> > > > did not have memory space to create an object.  Going
>> > > unresponsive
>> > > >> >> is
>> > > >> >> > > > not surprising.
>> > > >> >> > > >
>> > > >> >> > > > If you can solve your heap problem, note that you may run
>> > into
>> > > >> other
>> > > >> >> > > > performance issues discussed on the wiki page that I
>> linked.
>> > > >> >> > > >
>> > > >> >> > > > Thanks,
>> > > >> >> > > > Shawn
>> > > >> >> > > >
>> > > >> >> > >
>> > > >> >> >
>> > > >> >>
>> > > >> >
>> > > >> >
>> > > >>
>> > >
>> >
>>
>
>

Re: Solr server becomes non-responsive.

Posted by Modassar Ather <mo...@gmail.com>.

Thanks Jack for your suggestions.

Regards,
Modassar

On Fri, Dec 26, 2014 at 6:04 PM, Jack Krupansky <ja...@gmail.com>
wrote:

> Either you have too little RAM on each node or too much data on each node.
>
> You may need to shard the data much more heavily so that the total work on
> a single query is distributed in parallel to more nodes, each node having a
> much smaller amount of data to work on.
>
> First, always make sure that the entire Lucene index for each node fits
> entirely in the system memory available for file system caching. Otherwise
> the queries will be I/O bound. Check your current queries to see if that is
> the case - are the nodes compute bound or I/O bound? If I/O bound, add more
> system memory until the queries are no longer I/O bound. If compute bound,
> shard more heavily until the query latency becomes acceptable.
>
>
>
> -- Jack Krupansky
>
> On Fri, Dec 26, 2014 at 1:02 AM, Modassar Ather <mo...@gmail.com>
> wrote:
>
> > Thanks for your suggestions Erick.
> >
> > This may be one of those situations where you really have to
> > push back at the users and understand why they insist on these
> > kinds of queries. They must be very patient since it won't be
> > very performant. That said, I've seen this pattern; there are
> > certainly valid conditions under which response times can be
> > many seconds if there are few users and they are doing very
> > complex/expert-level things.
> >
> > We have tried educating the users but it did not work because they are
> used
> > to the old way. They feel that wildcard gives more control over the
> results
> > and may not fully understand stemming.
> >
> > Regards,
> > Modassar
> >
> > On Thu, Dec 25, 2014 at 3:17 AM, Erick Erickson <erickerickson@gmail.com
> >
> > wrote:
> >
> > > There's no magic bullet here that I know of. If your requirements
> > > are to support these huge, many-wildcard queries then you only
> > > have a few choices:
> > >
> > > 1> redo the index. I was surprised at how little it bloated the
> > > index as far as memory required is concerned to add ngrams.
> > > The key here is that there really aren't very many unique terms.
> > > If you use bigrams, then there are only maybe 36^2 distinct
> > > combinations. (assuming English and including numbers).
> > >
> > > 2> Increase the number of shards, putting many fewer docs
> > > on each shard.
> > >
> > > 3> give each shard a lot more memory. This isn't actually one
> > > of my preferred solutions since GC issues may raise their ugly
> > > heads here.
> > >
> > > 4> insert creative solution here.
> > >
> > > This may be one of those situations where you really have to
> > > push back at the users and understand why they insist on these
> > > kinds of queries. They must be very patient since it won't be
> > > very performant. That said, I've seen this pattern; there are
> > > certainly valid conditions under which response times can be
> > > many seconds if there are few users and they are doing very
> > > complex/expert-level things.
> > >
> > > Now, all that said, wildcards are often examples of poor habits
> > > or habits learned in DB systems where the only hammer was
> > > %whatever%. I've seen situations where users didn't
> > > understand that Solr broke the input stream up into words. And
> > > stemmed. And WordDelimiterFilterFactory did all the magic
> > > for finding, say D.C. and DC. So it's worth looking at the actual
> > > queries that are sent, perhaps talking to users and understanding
> > > what they _want_ out of the system, then perhaps educating them
> > > as to better ways to get what they want.
> > >
> > > Literally I've seen people insist on entering queries that
> > > wildcarded _everything_ both pre and post wildcards because
> > > they didn't realize that Solr tokenizes...
> > >
> > > Once you hit an OOM, all bets are off as Shawn outlined.
> > >
> > > Best,
> > > Erick
> > >
> > > On Wed, Dec 24, 2014 at 1:57 AM, Modassar Ather <
> modather1981@gmail.com>
> > > wrote:
> > > > Thanks for your response.
> > > >
> > > > How many items in the collection ?
> > > > There are about 100 millions documents.
> > > >
> > > > How are configured cache in solrconfig.xml ?
> > > > Each cache has size attribute as 128.
> > > >
> > > > Can you provide a sample of the query ?
> > > > Does it fail immediately after solrcloud startup or after several
> > hours ?
> > > > It is a query with many terms(more than a thousand) and phrase where
> > > > phrases have many wildcards in it.
> > > > Once such query is executed there are many zookeeper related
> exceptions
> > > and
> > > > with a couple of such queries executed it goes for OutOfMemory.
> > > >
> > > > Thanks,
> > > > Modassar
> > > >
> > > >
> > > > On Wed, Dec 24, 2014 at 1:37 PM, Dominique Bejean <
> > > dominique.bejean@eolya.fr
> > > >> wrote:
> > > >
> > > >> And you didn’t give how many RAM on each servers ?
> > > >>
> > > >> 2014-12-24 8:17 GMT+01:00 Dominique Bejean <
> dominique.bejean@eolya.fr
> > >:
> > > >>
> > > >> > Modassar,
> > > >> >
> > > >> > How many items in the collection ?
> > > >> > I mean how many documents per collection ? 1 million, 10 millions,
> > …?
> > > >> >
> > > >> > How are configured cache in solrconfig.xml ?
> > > >> > What are the size attribute value for each cache ?
> > > >> >
> > > >> > Can you provide a sample of the query ?
> > > >> > Does it fail immediately after solrcloud startup or after several
> > > hours ?
> > > >> >
> > > >> > Dominique
> > > >> >
> > > >> > 2014-12-24 6:20 GMT+01:00 Modassar Ather <modather1981@gmail.com
> >:
> > > >> >
> > > >> >> Thanks for your suggestions.
> > > >> >>
> > > >> >> I will look into the link provided.
> > > >> >> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > > >> >>
> > > >> >> This is usually an anti-pattern. The very first thing
> > > >> >> I'd be doing is trying to not do this. See ngrams for infix
> > > >> >> queries, or shingles or ReverseWildcardFilterFactory or.....
> > > >> >>
> > > >> >> We cannot avoid multiple wildcards since that's is our user's
> > > >> requirement.
> > > >> >> We try to discourage it but the users insist on firing such
> > queries.
> > > >> Also,
> > > >> >> ngrams etc. can be tried but our index is already huge and ngrams
> > may
> > > >> >> further add lot to it. We are OK with such queries failing as
> long
> > as
> > > >> >> other
> > > >> >> queries are not affected.
> > > >> >>
> > > >> >>
> > > >> >> Please find the details below.
> > > >> >>
> > > >> >> So, how many nodes in the cluster ?
> > > >> >> There are total 4 nodes on the cluster.
> > > >> >>
> > > >> >> How many shards and replicas for the collection ?
> > > >> >> There are 4 shards and no replica for any of them.
> > > >> >>
> > > >> >> How many items in the collection ?
> > > >> >> If I understand the question correctly there are two collection
> on
> > > each
> > > >> >> node and there size on each node is approximately 190GB and
> 130GB.
> > > >> >>
> > > >> >> What is the size of the index ?
> > > >> >> There are two collection on each node and there size on each node
> > is
> > > >> >> approximately 190GB and 130GB.
> > > >> >>
> > > >> >> How is updated the collection (frequency, how many items per
> days,
> > > what
> > > >> is
> > > >> >> your hard commit strategy) ?
> > > >> >> It is an optimized index and read-only. There are no
> inter-mediate
> > > >> update.
> > > >> >>
> > > >> >> How are configured cache in solrconfig.xml ?
> > > >> >> Filter cache, query result cache and document cache are enabled.
> > > >> >> Auto-warming is also done.
> > > >> >>
> > > >> >> Can you provide all other JVM parameters ?
> > > >> >> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
> > > >> >>
> > > >> >> Thanks again,
> > > >> >> Modassar
> > > >> >>
> > > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> > > >> >> dominique.bejean@eolya.fr
> > > >> >> > wrote:
> > > >> >>
> > > >> >> > Hi,
> > > >> >> >
> > > >> >> > I agree Erick it could be a good think to have more details
> about
> > > your
> > > >> >> > configuration and collection.
> > > >> >> >
> > > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> > > >> >> >
> > > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers
> or a
> > > >> >> > collection with 4 shards ?
> > > >> >> >
> > > >> >> > So, how many nodes in the cluster ?
> > > >> >> > How many shards and replicas for the collection ?
> > > >> >> > How many items in the collection ?
> > > >> >> > What is the size of the index ?
> > > >> >> > How is updated the collection (frequency, how many items per
> > days,
> > > >> what
> > > >> >> is
> > > >> >> > your hard commit strategy) ?
> > > >> >> > How are configured cache in solrconfig.xml ?
> > > >> >> > Can you provide all other JVM parameters ?
> > > >> >> >
> > > >> >> > Regards
> > > >> >> >
> > > >> >> > Dominique
> > > >> >> >
> > > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
> > erickerickson@gmail.com
> > > >:
> > > >> >> >
> > > >> >> > > Second most important part of your message:
> > > >> >> > > "When executing a huge query with many wildcards inside it
> the
> > > >> server"
> > > >> >> > >
> > > >> >> > > This is usually an anti-pattern. The very first thing
> > > >> >> > > I'd be doing is trying to not do this. See ngrams for infix
> > > >> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> > > >> >> > >
> > > >> >> > > And if your corpus is very large with many unique terms it's
> > even
> > > >> >> > > worse, but you haven't really told us about that yet.
> > > >> >> > >
> > > >> >> > > Best,
> > > >> >> > > Erick
> > > >> >> > >
> > > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> > > apache@elyograg.org>
> > > >> >> > wrote:
> > > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > > >> >> > > >> Hi,
> > > >> >> > > >>
> > > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> > > zookeeper on
> > > >> >> one
> > > >> >> > of
> > > >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is
> > 20g
> > > and
> > > >> >> -Xms
> > > >> >> > > is
> > > >> >> > > >> 24g.
> > > >> >> > > >> When executing a huge query with many wildcards inside it
> > the
> > > >> >> server
> > > >> >> > > >> crashes and becomes non-responsive. Even the dashboard
> does
> > > not
> > > >> >> > responds
> > > >> >> > > >> and shows connection lost error. This requires me to
> restart
> > > the
> > > >> >> > > servers.
> > > >> >> > > >
> > > >> >> > > > Here's the important part of your message:
> > > >> >> > > >
> > > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> > > >> >> > > >
> > > >> >> > > >
> > > >> >> > > > Your heap is not big enough for what Solr has been asked to
> > do.
> > > >> You
> > > >> >> > > > need to either increase your heap size or change your
> > > >> configuration
> > > >> >> so
> > > >> >> > > > that it uses less memory.
> > > >> >> > > >
> > > >> >> > > >
> > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > > >> >> > > >
> > > >> >> > > > Most programs have pretty much undefined behavior when an
> > OOME
> > > >> >> occurs.
> > > >> >> > > > Lucene's IndexWriter has been hardened so that it tries
> > > extremely
> > > >> >> hard
> > > >> >> > > > to avoid index corruption when OOME strikes, and I believe
> > that
> > > >> >> works
> > > >> >> > > > well enough that we can call it nearly bulletproof ... but
> > the
> > > >> rest
> > > >> >> of
> > > >> >> > > > Lucene and Solr will make no guarantees.
> > > >> >> > > >
> > > >> >> > > > It's very difficult to have definable program behavior when
> > an
> > > >> OOME
> > > >> >> > > > happens, because you simply cannot know the precise point
> > > during
> > > >> >> > program
> > > >> >> > > > execution where it will happen, or what isn't going to work
> > > >> because
> > > >> >> > Java
> > > >> >> > > > did not have memory space to create an object.  Going
> > > unresponsive
> > > >> >> is
> > > >> >> > > > not surprising.
> > > >> >> > > >
> > > >> >> > > > If you can solve your heap problem, note that you may run
> > into
> > > >> other
> > > >> >> > > > performance issues discussed on the wiki page that I
> linked.
> > > >> >> > > >
> > > >> >> > > > Thanks,
> > > >> >> > > > Shawn
> > > >> >> > > >
> > > >> >> > >
> > > >> >> >
> > > >> >>
> > > >> >>
> > > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> > > >> >> dominique.bejean@eolya.fr
> > > >> >> > wrote:
> > > >> >>
> > > >> >> > Hi,
> > > >> >> >
> > > >> >> > I agree Erick it could be a good think to have more details
> about
> > > your
> > > >> >> > configuration and collection.
> > > >> >> >
> > > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> > > >> >> >
> > > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers
> or a
> > > >> >> > collection with 4 shards ?
> > > >> >> >
> > > >> >> > So, how many nodes in the cluster ?
> > > >> >> > How many shards and replicas for the collection ?
> > > >> >> > How many items in the collection ?
> > > >> >> > What is the size of the index ?
> > > >> >> > How is updated the collection (frequency, how many items per
> > days,
> > > >> what
> > > >> >> is
> > > >> >> > your hard commit strategy) ?
> > > >> >> > How are configured cache in solrconfig.xml ?
> > > >> >> > Can you provide all other JVM parameters ?
> > > >> >> >
> > > >> >> > Regards
> > > >> >> >
> > > >> >> > Dominique
> > > >> >> >
> > > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
> > erickerickson@gmail.com
> > > >:
> > > >> >> >
> > > >> >> > > Second most important part of your message:
> > > >> >> > > "When executing a huge query with many wildcards inside it
> the
> > > >> server"
> > > >> >> > >
> > > >> >> > > This is usually an anti-pattern. The very first thing
> > > >> >> > > I'd be doing is trying to not do this. See ngrams for infix
> > > >> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> > > >> >> > >
> > > >> >> > > And if your corpus is very large with many unique terms it's
> > even
> > > >> >> > > worse, but you haven't really told us about that yet.
> > > >> >> > >
> > > >> >> > > Best,
> > > >> >> > > Erick
> > > >> >> > >
> > > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> > > apache@elyograg.org>
> > > >> >> > wrote:
> > > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > > >> >> > > >> Hi,
> > > >> >> > > >>
> > > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> > > zookeeper on
> > > >> >> one
> > > >> >> > of
> > > >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is
> > 20g
> > > and
> > > >> >> -Xms
> > > >> >> > > is
> > > >> >> > > >> 24g.
> > > >> >> > > >> When executing a huge query with many wildcards inside it
> > the
> > > >> >> server
> > > >> >> > > >> crashes and becomes non-responsive. Even the dashboard
> does
> > > not
> > > >> >> > responds
> > > >> >> > > >> and shows connection lost error. This requires me to
> restart
> > > the
> > > >> >> > > servers.
> > > >> >> > > >
> > > >> >> > > > Here's the important part of your message:
> > > >> >> > > >
> > > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> > > >> >> > > >
> > > >> >> > > >
> > > >> >> > > > Your heap is not big enough for what Solr has been asked to
> > do.
> > > >> You
> > > >> >> > > > need to either increase your heap size or change your
> > > >> configuration
> > > >> >> so
> > > >> >> > > > that it uses less memory.
> > > >> >> > > >
> > > >> >> > > >
> > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > > >> >> > > >
> > > >> >> > > > Most programs have pretty much undefined behavior when an
> > OOME
> > > >> >> occurs.
> > > >> >> > > > Lucene's IndexWriter has been hardened so that it tries
> > > extremely
> > > >> >> hard
> > > >> >> > > > to avoid index corruption when OOME strikes, and I believe
> > that
> > > >> >> works
> > > >> >> > > > well enough that we can call it nearly bulletproof ... but
> > the
> > > >> rest
> > > >> >> of
> > > >> >> > > > Lucene and Solr will make no guarantees.
> > > >> >> > > >
> > > >> >> > > > It's very difficult to have definable program behavior when
> > an
> > > >> OOME
> > > >> >> > > > happens, because you simply cannot know the precise point
> > > during
> > > >> >> > program
> > > >> >> > > > execution where it will happen, or what isn't going to work
> > > >> because
> > > >> >> > Java
> > > >> >> > > > did not have memory space to create an object.  Going
> > > unresponsive
> > > >> >> is
> > > >> >> > > > not surprising.
> > > >> >> > > >
> > > >> >> > > > If you can solve your heap problem, note that you may run
> > into
> > > >> other
> > > >> >> > > > performance issues discussed on the wiki page that I
> linked.
> > > >> >> > > >
> > > >> >> > > > Thanks,
> > > >> >> > > > Shawn
> > > >> >> > > >
> > > >> >> > >
> > > >> >> >
> > > >> >>
> > > >> >
> > > >> >
> > > >>
> > >
> >
>

Re: Solr server becomes non-responsive.

Posted by Jack Krupansky <ja...@gmail.com>.

Either you have too little RAM on each node or too much data on each node.

You may need to shard the data much more heavily so that the total work on
a single query is distributed in parallel to more nodes, each node having a
much smaller amount of data to work on.

First, always make sure that the entire Lucene index for each node fits
entirely in the system memory available for file system caching. Otherwise
the queries will be I/O bound. Check your current queries to see if that is
the case - are the nodes compute bound or I/O bound? If I/O bound, add more
system memory until the queries are no longer I/O bound. If compute bound,
shard more heavily until the query latency becomes acceptable.



-- Jack Krupansky

On Fri, Dec 26, 2014 at 1:02 AM, Modassar Ather <mo...@gmail.com>
wrote:

> Thanks for your suggestions Erick.
>
> This may be one of those situations where you really have to
> push back at the users and understand why they insist on these
> kinds of queries. They must be very patient since it won't be
> very performant. That said, I've seen this pattern; there are
> certainly valid conditions under which response times can be
> many seconds if there are few users and they are doing very
> complex/expert-level things.
>
> We have tried educating the users but it did not work because they are used
> to the old way. They feel that wildcard gives more control over the results
> and may not fully understand stemming.
>
> Regards,
> Modassar
>
> On Thu, Dec 25, 2014 at 3:17 AM, Erick Erickson <er...@gmail.com>
> wrote:
>
> > There's no magic bullet here that I know of. If your requirements
> > are to support these huge, many-wildcard queries then you only
> > have a few choices:
> >
> > 1> redo the index. I was surprised at how little it bloated the
> > index as far as memory required is concerned to add ngrams.
> > The key here is that there really aren't very many unique terms.
> > If you use bigrams, then there are only maybe 36^2 distinct
> > combinations. (assuming English and including numbers).
> >
> > 2> Increase the number of shards, putting many fewer docs
> > on each shard.
> >
> > 3> give each shard a lot more memory. This isn't actually one
> > of my preferred solutions since GC issues may raise their ugly
> > heads here.
> >
> > 4> insert creative solution here.
> >
> > This may be one of those situations where you really have to
> > push back at the users and understand why they insist on these
> > kinds of queries. They must be very patient since it won't be
> > very performant. That said, I've seen this pattern; there are
> > certainly valid conditions under which response times can be
> > many seconds if there are few users and they are doing very
> > complex/expert-level things.
> >
> > Now, all that said, wildcards are often examples of poor habits
> > or habits learned in DB systems where the only hammer was
> > %whatever%. I've seen situations where users didn't
> > understand that Solr broke the input stream up into words. And
> > stemmed. And WordDelimiterFilterFactory did all the magic
> > for finding, say D.C. and DC. So it's worth looking at the actual
> > queries that are sent, perhaps talking to users and understanding
> > what they _want_ out of the system, then perhaps educating them
> > as to better ways to get what they want.
> >
> > Literally I've seen people insist on entering queries that
> > wildcarded _everything_ both pre and post wildcards because
> > they didn't realize that Solr tokenizes...
> >
> > Once you hit an OOM, all bets are off as Shawn outlined.
> >
> > Best,
> > Erick
> >
> > On Wed, Dec 24, 2014 at 1:57 AM, Modassar Ather <mo...@gmail.com>
> > wrote:
> > > Thanks for your response.
> > >
> > > How many items in the collection ?
> > > There are about 100 millions documents.
> > >
> > > How are configured cache in solrconfig.xml ?
> > > Each cache has size attribute as 128.
> > >
> > > Can you provide a sample of the query ?
> > > Does it fail immediately after solrcloud startup or after several
> hours ?
> > > It is a query with many terms(more than a thousand) and phrase where
> > > phrases have many wildcards in it.
> > > Once such query is executed there are many zookeeper related exceptions
> > and
> > > with a couple of such queries executed it goes for OutOfMemory.
> > >
> > > Thanks,
> > > Modassar
> > >
> > >
> > > On Wed, Dec 24, 2014 at 1:37 PM, Dominique Bejean <
> > dominique.bejean@eolya.fr
> > >> wrote:
> > >
> > >> And you didn’t give how many RAM on each servers ?
> > >>
> > >> 2014-12-24 8:17 GMT+01:00 Dominique Bejean <dominique.bejean@eolya.fr
> >:
> > >>
> > >> > Modassar,
> > >> >
> > >> > How many items in the collection ?
> > >> > I mean how many documents per collection ? 1 million, 10 millions,
> …?
> > >> >
> > >> > How are configured cache in solrconfig.xml ?
> > >> > What are the size attribute value for each cache ?
> > >> >
> > >> > Can you provide a sample of the query ?
> > >> > Does it fail immediately after solrcloud startup or after several
> > hours ?
> > >> >
> > >> > Dominique
> > >> >
> > >> > 2014-12-24 6:20 GMT+01:00 Modassar Ather <mo...@gmail.com>:
> > >> >
> > >> >> Thanks for your suggestions.
> > >> >>
> > >> >> I will look into the link provided.
> > >> >> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > >> >>
> > >> >> This is usually an anti-pattern. The very first thing
> > >> >> I'd be doing is trying to not do this. See ngrams for infix
> > >> >> queries, or shingles or ReverseWildcardFilterFactory or.....
> > >> >>
> > >> >> We cannot avoid multiple wildcards since that's is our user's
> > >> requirement.
> > >> >> We try to discourage it but the users insist on firing such
> queries.
> > >> Also,
> > >> >> ngrams etc. can be tried but our index is already huge and ngrams
> may
> > >> >> further add lot to it. We are OK with such queries failing as long
> as
> > >> >> other
> > >> >> queries are not affected.
> > >> >>
> > >> >>
> > >> >> Please find the details below.
> > >> >>
> > >> >> So, how many nodes in the cluster ?
> > >> >> There are total 4 nodes on the cluster.
> > >> >>
> > >> >> How many shards and replicas for the collection ?
> > >> >> There are 4 shards and no replica for any of them.
> > >> >>
> > >> >> How many items in the collection ?
> > >> >> If I understand the question correctly there are two collection on
> > each
> > >> >> node and there size on each node is approximately 190GB and 130GB.
> > >> >>
> > >> >> What is the size of the index ?
> > >> >> There are two collection on each node and there size on each node
> is
> > >> >> approximately 190GB and 130GB.
> > >> >>
> > >> >> How is updated the collection (frequency, how many items per days,
> > what
> > >> is
> > >> >> your hard commit strategy) ?
> > >> >> It is an optimized index and read-only. There are no inter-mediate
> > >> update.
> > >> >>
> > >> >> How are configured cache in solrconfig.xml ?
> > >> >> Filter cache, query result cache and document cache are enabled.
> > >> >> Auto-warming is also done.
> > >> >>
> > >> >> Can you provide all other JVM parameters ?
> > >> >> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
> > >> >>
> > >> >> Thanks again,
> > >> >> Modassar
> > >> >>
> > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> > >> >> dominique.bejean@eolya.fr
> > >> >> > wrote:
> > >> >>
> > >> >> > Hi,
> > >> >> >
> > >> >> > I agree Erick it could be a good think to have more details about
> > your
> > >> >> > configuration and collection.
> > >> >> >
> > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> > >> >> >
> > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> > >> >> > collection with 4 shards ?
> > >> >> >
> > >> >> > So, how many nodes in the cluster ?
> > >> >> > How many shards and replicas for the collection ?
> > >> >> > How many items in the collection ?
> > >> >> > What is the size of the index ?
> > >> >> > How is updated the collection (frequency, how many items per
> days,
> > >> what
> > >> >> is
> > >> >> > your hard commit strategy) ?
> > >> >> > How are configured cache in solrconfig.xml ?
> > >> >> > Can you provide all other JVM parameters ?
> > >> >> >
> > >> >> > Regards
> > >> >> >
> > >> >> > Dominique
> > >> >> >
> > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
> erickerickson@gmail.com
> > >:
> > >> >> >
> > >> >> > > Second most important part of your message:
> > >> >> > > "When executing a huge query with many wildcards inside it the
> > >> server"
> > >> >> > >
> > >> >> > > This is usually an anti-pattern. The very first thing
> > >> >> > > I'd be doing is trying to not do this. See ngrams for infix
> > >> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> > >> >> > >
> > >> >> > > And if your corpus is very large with many unique terms it's
> even
> > >> >> > > worse, but you haven't really told us about that yet.
> > >> >> > >
> > >> >> > > Best,
> > >> >> > > Erick
> > >> >> > >
> > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> > apache@elyograg.org>
> > >> >> > wrote:
> > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > >> >> > > >> Hi,
> > >> >> > > >>
> > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> > zookeeper on
> > >> >> one
> > >> >> > of
> > >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is
> 20g
> > and
> > >> >> -Xms
> > >> >> > > is
> > >> >> > > >> 24g.
> > >> >> > > >> When executing a huge query with many wildcards inside it
> the
> > >> >> server
> > >> >> > > >> crashes and becomes non-responsive. Even the dashboard does
> > not
> > >> >> > responds
> > >> >> > > >> and shows connection lost error. This requires me to restart
> > the
> > >> >> > > servers.
> > >> >> > > >
> > >> >> > > > Here's the important part of your message:
> > >> >> > > >
> > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> > >> >> > > >
> > >> >> > > >
> > >> >> > > > Your heap is not big enough for what Solr has been asked to
> do.
> > >> You
> > >> >> > > > need to either increase your heap size or change your
> > >> configuration
> > >> >> so
> > >> >> > > > that it uses less memory.
> > >> >> > > >
> > >> >> > > >
> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > >> >> > > >
> > >> >> > > > Most programs have pretty much undefined behavior when an
> OOME
> > >> >> occurs.
> > >> >> > > > Lucene's IndexWriter has been hardened so that it tries
> > extremely
> > >> >> hard
> > >> >> > > > to avoid index corruption when OOME strikes, and I believe
> that
> > >> >> works
> > >> >> > > > well enough that we can call it nearly bulletproof ... but
> the
> > >> rest
> > >> >> of
> > >> >> > > > Lucene and Solr will make no guarantees.
> > >> >> > > >
> > >> >> > > > It's very difficult to have definable program behavior when
> an
> > >> OOME
> > >> >> > > > happens, because you simply cannot know the precise point
> > during
> > >> >> > program
> > >> >> > > > execution where it will happen, or what isn't going to work
> > >> because
> > >> >> > Java
> > >> >> > > > did not have memory space to create an object.  Going
> > unresponsive
> > >> >> is
> > >> >> > > > not surprising.
> > >> >> > > >
> > >> >> > > > If you can solve your heap problem, note that you may run
> into
> > >> other
> > >> >> > > > performance issues discussed on the wiki page that I linked.
> > >> >> > > >
> > >> >> > > > Thanks,
> > >> >> > > > Shawn
> > >> >> > > >
> > >> >> > >
> > >> >> >
> > >> >>
> > >> >>
> > >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> > >> >> dominique.bejean@eolya.fr
> > >> >> > wrote:
> > >> >>
> > >> >> > Hi,
> > >> >> >
> > >> >> > I agree Erick it could be a good think to have more details about
> > your
> > >> >> > configuration and collection.
> > >> >> >
> > >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> > >> >> >
> > >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> > >> >> > collection with 4 shards ?
> > >> >> >
> > >> >> > So, how many nodes in the cluster ?
> > >> >> > How many shards and replicas for the collection ?
> > >> >> > How many items in the collection ?
> > >> >> > What is the size of the index ?
> > >> >> > How is updated the collection (frequency, how many items per
> days,
> > >> what
> > >> >> is
> > >> >> > your hard commit strategy) ?
> > >> >> > How are configured cache in solrconfig.xml ?
> > >> >> > Can you provide all other JVM parameters ?
> > >> >> >
> > >> >> > Regards
> > >> >> >
> > >> >> > Dominique
> > >> >> >
> > >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <
> erickerickson@gmail.com
> > >:
> > >> >> >
> > >> >> > > Second most important part of your message:
> > >> >> > > "When executing a huge query with many wildcards inside it the
> > >> server"
> > >> >> > >
> > >> >> > > This is usually an anti-pattern. The very first thing
> > >> >> > > I'd be doing is trying to not do this. See ngrams for infix
> > >> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> > >> >> > >
> > >> >> > > And if your corpus is very large with many unique terms it's
> even
> > >> >> > > worse, but you haven't really told us about that yet.
> > >> >> > >
> > >> >> > > Best,
> > >> >> > > Erick
> > >> >> > >
> > >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> > apache@elyograg.org>
> > >> >> > wrote:
> > >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > >> >> > > >> Hi,
> > >> >> > > >>
> > >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> > zookeeper on
> > >> >> one
> > >> >> > of
> > >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is
> 20g
> > and
> > >> >> -Xms
> > >> >> > > is
> > >> >> > > >> 24g.
> > >> >> > > >> When executing a huge query with many wildcards inside it
> the
> > >> >> server
> > >> >> > > >> crashes and becomes non-responsive. Even the dashboard does
> > not
> > >> >> > responds
> > >> >> > > >> and shows connection lost error. This requires me to restart
> > the
> > >> >> > > servers.
> > >> >> > > >
> > >> >> > > > Here's the important part of your message:
> > >> >> > > >
> > >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> > >> >> > > >
> > >> >> > > >
> > >> >> > > > Your heap is not big enough for what Solr has been asked to
> do.
> > >> You
> > >> >> > > > need to either increase your heap size or change your
> > >> configuration
> > >> >> so
> > >> >> > > > that it uses less memory.
> > >> >> > > >
> > >> >> > > >
> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > >> >> > > >
> > >> >> > > > Most programs have pretty much undefined behavior when an
> OOME
> > >> >> occurs.
> > >> >> > > > Lucene's IndexWriter has been hardened so that it tries
> > extremely
> > >> >> hard
> > >> >> > > > to avoid index corruption when OOME strikes, and I believe
> that
> > >> >> works
> > >> >> > > > well enough that we can call it nearly bulletproof ... but
> the
> > >> rest
> > >> >> of
> > >> >> > > > Lucene and Solr will make no guarantees.
> > >> >> > > >
> > >> >> > > > It's very difficult to have definable program behavior when
> an
> > >> OOME
> > >> >> > > > happens, because you simply cannot know the precise point
> > during
> > >> >> > program
> > >> >> > > > execution where it will happen, or what isn't going to work
> > >> because
> > >> >> > Java
> > >> >> > > > did not have memory space to create an object.  Going
> > unresponsive
> > >> >> is
> > >> >> > > > not surprising.
> > >> >> > > >
> > >> >> > > > If you can solve your heap problem, note that you may run
> into
> > >> other
> > >> >> > > > performance issues discussed on the wiki page that I linked.
> > >> >> > > >
> > >> >> > > > Thanks,
> > >> >> > > > Shawn
> > >> >> > > >
> > >> >> > >
> > >> >> >
> > >> >>
> > >> >
> > >> >
> > >>
> >
>

Re: Solr server becomes non-responsive.

Posted by Modassar Ather <mo...@gmail.com>.

Thanks for your suggestions Erick.

This may be one of those situations where you really have to
push back at the users and understand why they insist on these
kinds of queries. They must be very patient since it won't be
very performant. That said, I've seen this pattern; there are
certainly valid conditions under which response times can be
many seconds if there are few users and they are doing very
complex/expert-level things.

We have tried educating the users but it did not work because they are used
to the old way. They feel that wildcard gives more control over the results
and may not fully understand stemming.

Regards,
Modassar

On Thu, Dec 25, 2014 at 3:17 AM, Erick Erickson <er...@gmail.com>
wrote:

> There's no magic bullet here that I know of. If your requirements
> are to support these huge, many-wildcard queries then you only
> have a few choices:
>
> 1> redo the index. I was surprised at how little it bloated the
> index as far as memory required is concerned to add ngrams.
> The key here is that there really aren't very many unique terms.
> If you use bigrams, then there are only maybe 36^2 distinct
> combinations. (assuming English and including numbers).
>
> 2> Increase the number of shards, putting many fewer docs
> on each shard.
>
> 3> give each shard a lot more memory. This isn't actually one
> of my preferred solutions since GC issues may raise their ugly
> heads here.
>
> 4> insert creative solution here.
>
> This may be one of those situations where you really have to
> push back at the users and understand why they insist on these
> kinds of queries. They must be very patient since it won't be
> very performant. That said, I've seen this pattern; there are
> certainly valid conditions under which response times can be
> many seconds if there are few users and they are doing very
> complex/expert-level things.
>
> Now, all that said, wildcards are often examples of poor habits
> or habits learned in DB systems where the only hammer was
> %whatever%. I've seen situations where users didn't
> understand that Solr broke the input stream up into words. And
> stemmed. And WordDelimiterFilterFactory did all the magic
> for finding, say D.C. and DC. So it's worth looking at the actual
> queries that are sent, perhaps talking to users and understanding
> what they _want_ out of the system, then perhaps educating them
> as to better ways to get what they want.
>
> Literally I've seen people insist on entering queries that
> wildcarded _everything_ both pre and post wildcards because
> they didn't realize that Solr tokenizes...
>
> Once you hit an OOM, all bets are off as Shawn outlined.
>
> Best,
> Erick
>
> On Wed, Dec 24, 2014 at 1:57 AM, Modassar Ather <mo...@gmail.com>
> wrote:
> > Thanks for your response.
> >
> > How many items in the collection ?
> > There are about 100 millions documents.
> >
> > How are configured cache in solrconfig.xml ?
> > Each cache has size attribute as 128.
> >
> > Can you provide a sample of the query ?
> > Does it fail immediately after solrcloud startup or after several hours ?
> > It is a query with many terms(more than a thousand) and phrase where
> > phrases have many wildcards in it.
> > Once such query is executed there are many zookeeper related exceptions
> and
> > with a couple of such queries executed it goes for OutOfMemory.
> >
> > Thanks,
> > Modassar
> >
> >
> > On Wed, Dec 24, 2014 at 1:37 PM, Dominique Bejean <
> dominique.bejean@eolya.fr
> >> wrote:
> >
> >> And you didn’t give how many RAM on each servers ?
> >>
> >> 2014-12-24 8:17 GMT+01:00 Dominique Bejean <do...@eolya.fr>:
> >>
> >> > Modassar,
> >> >
> >> > How many items in the collection ?
> >> > I mean how many documents per collection ? 1 million, 10 millions, …?
> >> >
> >> > How are configured cache in solrconfig.xml ?
> >> > What are the size attribute value for each cache ?
> >> >
> >> > Can you provide a sample of the query ?
> >> > Does it fail immediately after solrcloud startup or after several
> hours ?
> >> >
> >> > Dominique
> >> >
> >> > 2014-12-24 6:20 GMT+01:00 Modassar Ather <mo...@gmail.com>:
> >> >
> >> >> Thanks for your suggestions.
> >> >>
> >> >> I will look into the link provided.
> >> >> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >> >>
> >> >> This is usually an anti-pattern. The very first thing
> >> >> I'd be doing is trying to not do this. See ngrams for infix
> >> >> queries, or shingles or ReverseWildcardFilterFactory or.....
> >> >>
> >> >> We cannot avoid multiple wildcards since that's is our user's
> >> requirement.
> >> >> We try to discourage it but the users insist on firing such queries.
> >> Also,
> >> >> ngrams etc. can be tried but our index is already huge and ngrams may
> >> >> further add lot to it. We are OK with such queries failing as long as
> >> >> other
> >> >> queries are not affected.
> >> >>
> >> >>
> >> >> Please find the details below.
> >> >>
> >> >> So, how many nodes in the cluster ?
> >> >> There are total 4 nodes on the cluster.
> >> >>
> >> >> How many shards and replicas for the collection ?
> >> >> There are 4 shards and no replica for any of them.
> >> >>
> >> >> How many items in the collection ?
> >> >> If I understand the question correctly there are two collection on
> each
> >> >> node and there size on each node is approximately 190GB and 130GB.
> >> >>
> >> >> What is the size of the index ?
> >> >> There are two collection on each node and there size on each node is
> >> >> approximately 190GB and 130GB.
> >> >>
> >> >> How is updated the collection (frequency, how many items per days,
> what
> >> is
> >> >> your hard commit strategy) ?
> >> >> It is an optimized index and read-only. There are no inter-mediate
> >> update.
> >> >>
> >> >> How are configured cache in solrconfig.xml ?
> >> >> Filter cache, query result cache and document cache are enabled.
> >> >> Auto-warming is also done.
> >> >>
> >> >> Can you provide all other JVM parameters ?
> >> >> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
> >> >>
> >> >> Thanks again,
> >> >> Modassar
> >> >>
> >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> >> >> dominique.bejean@eolya.fr
> >> >> > wrote:
> >> >>
> >> >> > Hi,
> >> >> >
> >> >> > I agree Erick it could be a good think to have more details about
> your
> >> >> > configuration and collection.
> >> >> >
> >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> >> >> >
> >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> >> >> > collection with 4 shards ?
> >> >> >
> >> >> > So, how many nodes in the cluster ?
> >> >> > How many shards and replicas for the collection ?
> >> >> > How many items in the collection ?
> >> >> > What is the size of the index ?
> >> >> > How is updated the collection (frequency, how many items per days,
> >> what
> >> >> is
> >> >> > your hard commit strategy) ?
> >> >> > How are configured cache in solrconfig.xml ?
> >> >> > Can you provide all other JVM parameters ?
> >> >> >
> >> >> > Regards
> >> >> >
> >> >> > Dominique
> >> >> >
> >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <erickerickson@gmail.com
> >:
> >> >> >
> >> >> > > Second most important part of your message:
> >> >> > > "When executing a huge query with many wildcards inside it the
> >> server"
> >> >> > >
> >> >> > > This is usually an anti-pattern. The very first thing
> >> >> > > I'd be doing is trying to not do this. See ngrams for infix
> >> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> >> >> > >
> >> >> > > And if your corpus is very large with many unique terms it's even
> >> >> > > worse, but you haven't really told us about that yet.
> >> >> > >
> >> >> > > Best,
> >> >> > > Erick
> >> >> > >
> >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> apache@elyograg.org>
> >> >> > wrote:
> >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> >> >> > > >> Hi,
> >> >> > > >>
> >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> zookeeper on
> >> >> one
> >> >> > of
> >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g
> and
> >> >> -Xms
> >> >> > > is
> >> >> > > >> 24g.
> >> >> > > >> When executing a huge query with many wildcards inside it the
> >> >> server
> >> >> > > >> crashes and becomes non-responsive. Even the dashboard does
> not
> >> >> > responds
> >> >> > > >> and shows connection lost error. This requires me to restart
> the
> >> >> > > servers.
> >> >> > > >
> >> >> > > > Here's the important part of your message:
> >> >> > > >
> >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> >> >> > > >
> >> >> > > >
> >> >> > > > Your heap is not big enough for what Solr has been asked to do.
> >> You
> >> >> > > > need to either increase your heap size or change your
> >> configuration
> >> >> so
> >> >> > > > that it uses less memory.
> >> >> > > >
> >> >> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >> >> > > >
> >> >> > > > Most programs have pretty much undefined behavior when an OOME
> >> >> occurs.
> >> >> > > > Lucene's IndexWriter has been hardened so that it tries
> extremely
> >> >> hard
> >> >> > > > to avoid index corruption when OOME strikes, and I believe that
> >> >> works
> >> >> > > > well enough that we can call it nearly bulletproof ... but the
> >> rest
> >> >> of
> >> >> > > > Lucene and Solr will make no guarantees.
> >> >> > > >
> >> >> > > > It's very difficult to have definable program behavior when an
> >> OOME
> >> >> > > > happens, because you simply cannot know the precise point
> during
> >> >> > program
> >> >> > > > execution where it will happen, or what isn't going to work
> >> because
> >> >> > Java
> >> >> > > > did not have memory space to create an object.  Going
> unresponsive
> >> >> is
> >> >> > > > not surprising.
> >> >> > > >
> >> >> > > > If you can solve your heap problem, note that you may run into
> >> other
> >> >> > > > performance issues discussed on the wiki page that I linked.
> >> >> > > >
> >> >> > > > Thanks,
> >> >> > > > Shawn
> >> >> > > >
> >> >> > >
> >> >> >
> >> >>
> >> >>
> >> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> >> >> dominique.bejean@eolya.fr
> >> >> > wrote:
> >> >>
> >> >> > Hi,
> >> >> >
> >> >> > I agree Erick it could be a good think to have more details about
> your
> >> >> > configuration and collection.
> >> >> >
> >> >> > Your heap size is 32Gb. How many RAM on each servers ?
> >> >> >
> >> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> >> >> > collection with 4 shards ?
> >> >> >
> >> >> > So, how many nodes in the cluster ?
> >> >> > How many shards and replicas for the collection ?
> >> >> > How many items in the collection ?
> >> >> > What is the size of the index ?
> >> >> > How is updated the collection (frequency, how many items per days,
> >> what
> >> >> is
> >> >> > your hard commit strategy) ?
> >> >> > How are configured cache in solrconfig.xml ?
> >> >> > Can you provide all other JVM parameters ?
> >> >> >
> >> >> > Regards
> >> >> >
> >> >> > Dominique
> >> >> >
> >> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <erickerickson@gmail.com
> >:
> >> >> >
> >> >> > > Second most important part of your message:
> >> >> > > "When executing a huge query with many wildcards inside it the
> >> server"
> >> >> > >
> >> >> > > This is usually an anti-pattern. The very first thing
> >> >> > > I'd be doing is trying to not do this. See ngrams for infix
> >> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> >> >> > >
> >> >> > > And if your corpus is very large with many unique terms it's even
> >> >> > > worse, but you haven't really told us about that yet.
> >> >> > >
> >> >> > > Best,
> >> >> > > Erick
> >> >> > >
> >> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <
> apache@elyograg.org>
> >> >> > wrote:
> >> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> >> >> > > >> Hi,
> >> >> > > >>
> >> >> > > >> I have a setup of 4 shard Solr cluster with embedded
> zookeeper on
> >> >> one
> >> >> > of
> >> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g
> and
> >> >> -Xms
> >> >> > > is
> >> >> > > >> 24g.
> >> >> > > >> When executing a huge query with many wildcards inside it the
> >> >> server
> >> >> > > >> crashes and becomes non-responsive. Even the dashboard does
> not
> >> >> > responds
> >> >> > > >> and shows connection lost error. This requires me to restart
> the
> >> >> > > servers.
> >> >> > > >
> >> >> > > > Here's the important part of your message:
> >> >> > > >
> >> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> >> >> > > >
> >> >> > > >
> >> >> > > > Your heap is not big enough for what Solr has been asked to do.
> >> You
> >> >> > > > need to either increase your heap size or change your
> >> configuration
> >> >> so
> >> >> > > > that it uses less memory.
> >> >> > > >
> >> >> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >> >> > > >
> >> >> > > > Most programs have pretty much undefined behavior when an OOME
> >> >> occurs.
> >> >> > > > Lucene's IndexWriter has been hardened so that it tries
> extremely
> >> >> hard
> >> >> > > > to avoid index corruption when OOME strikes, and I believe that
> >> >> works
> >> >> > > > well enough that we can call it nearly bulletproof ... but the
> >> rest
> >> >> of
> >> >> > > > Lucene and Solr will make no guarantees.
> >> >> > > >
> >> >> > > > It's very difficult to have definable program behavior when an
> >> OOME
> >> >> > > > happens, because you simply cannot know the precise point
> during
> >> >> > program
> >> >> > > > execution where it will happen, or what isn't going to work
> >> because
> >> >> > Java
> >> >> > > > did not have memory space to create an object.  Going
> unresponsive
> >> >> is
> >> >> > > > not surprising.
> >> >> > > >
> >> >> > > > If you can solve your heap problem, note that you may run into
> >> other
> >> >> > > > performance issues discussed on the wiki page that I linked.
> >> >> > > >
> >> >> > > > Thanks,
> >> >> > > > Shawn
> >> >> > > >
> >> >> > >
> >> >> >
> >> >>
> >> >
> >> >
> >>
>

Re: Solr server becomes non-responsive.

Posted by Erick Erickson <er...@gmail.com>.

There's no magic bullet here that I know of. If your requirements
are to support these huge, many-wildcard queries then you only
have a few choices:

1> redo the index. I was surprised at how little it bloated the
index as far as memory required is concerned to add ngrams.
The key here is that there really aren't very many unique terms.
If you use bigrams, then there are only maybe 36^2 distinct
combinations. (assuming English and including numbers).

2> Increase the number of shards, putting many fewer docs
on each shard.

3> give each shard a lot more memory. This isn't actually one
of my preferred solutions since GC issues may raise their ugly
heads here.

4> insert creative solution here.

This may be one of those situations where you really have to
push back at the users and understand why they insist on these
kinds of queries. They must be very patient since it won't be
very performant. That said, I've seen this pattern; there are
certainly valid conditions under which response times can be
many seconds if there are few users and they are doing very
complex/expert-level things.

Now, all that said, wildcards are often examples of poor habits
or habits learned in DB systems where the only hammer was
%whatever%. I've seen situations where users didn't
understand that Solr broke the input stream up into words. And
stemmed. And WordDelimiterFilterFactory did all the magic
for finding, say D.C. and DC. So it's worth looking at the actual
queries that are sent, perhaps talking to users and understanding
what they _want_ out of the system, then perhaps educating them
as to better ways to get what they want.

Literally I've seen people insist on entering queries that
wildcarded _everything_ both pre and post wildcards because
they didn't realize that Solr tokenizes...

Once you hit an OOM, all bets are off as Shawn outlined.

Best,
Erick

On Wed, Dec 24, 2014 at 1:57 AM, Modassar Ather <mo...@gmail.com> wrote:
> Thanks for your response.
>
> How many items in the collection ?
> There are about 100 millions documents.
>
> How are configured cache in solrconfig.xml ?
> Each cache has size attribute as 128.
>
> Can you provide a sample of the query ?
> Does it fail immediately after solrcloud startup or after several hours ?
> It is a query with many terms(more than a thousand) and phrase where
> phrases have many wildcards in it.
> Once such query is executed there are many zookeeper related exceptions and
> with a couple of such queries executed it goes for OutOfMemory.
>
> Thanks,
> Modassar
>
>
> On Wed, Dec 24, 2014 at 1:37 PM, Dominique Bejean <dominique.bejean@eolya.fr
>> wrote:
>
>> And you didn’t give how many RAM on each servers ?
>>
>> 2014-12-24 8:17 GMT+01:00 Dominique Bejean <do...@eolya.fr>:
>>
>> > Modassar,
>> >
>> > How many items in the collection ?
>> > I mean how many documents per collection ? 1 million, 10 millions, …?
>> >
>> > How are configured cache in solrconfig.xml ?
>> > What are the size attribute value for each cache ?
>> >
>> > Can you provide a sample of the query ?
>> > Does it fail immediately after solrcloud startup or after several hours ?
>> >
>> > Dominique
>> >
>> > 2014-12-24 6:20 GMT+01:00 Modassar Ather <mo...@gmail.com>:
>> >
>> >> Thanks for your suggestions.
>> >>
>> >> I will look into the link provided.
>> >> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>> >>
>> >> This is usually an anti-pattern. The very first thing
>> >> I'd be doing is trying to not do this. See ngrams for infix
>> >> queries, or shingles or ReverseWildcardFilterFactory or.....
>> >>
>> >> We cannot avoid multiple wildcards since that's is our user's
>> requirement.
>> >> We try to discourage it but the users insist on firing such queries.
>> Also,
>> >> ngrams etc. can be tried but our index is already huge and ngrams may
>> >> further add lot to it. We are OK with such queries failing as long as
>> >> other
>> >> queries are not affected.
>> >>
>> >>
>> >> Please find the details below.
>> >>
>> >> So, how many nodes in the cluster ?
>> >> There are total 4 nodes on the cluster.
>> >>
>> >> How many shards and replicas for the collection ?
>> >> There are 4 shards and no replica for any of them.
>> >>
>> >> How many items in the collection ?
>> >> If I understand the question correctly there are two collection on each
>> >> node and there size on each node is approximately 190GB and 130GB.
>> >>
>> >> What is the size of the index ?
>> >> There are two collection on each node and there size on each node is
>> >> approximately 190GB and 130GB.
>> >>
>> >> How is updated the collection (frequency, how many items per days, what
>> is
>> >> your hard commit strategy) ?
>> >> It is an optimized index and read-only. There are no inter-mediate
>> update.
>> >>
>> >> How are configured cache in solrconfig.xml ?
>> >> Filter cache, query result cache and document cache are enabled.
>> >> Auto-warming is also done.
>> >>
>> >> Can you provide all other JVM parameters ?
>> >> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
>> >>
>> >> Thanks again,
>> >> Modassar
>> >>
>> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
>> >> dominique.bejean@eolya.fr
>> >> > wrote:
>> >>
>> >> > Hi,
>> >> >
>> >> > I agree Erick it could be a good think to have more details about your
>> >> > configuration and collection.
>> >> >
>> >> > Your heap size is 32Gb. How many RAM on each servers ?
>> >> >
>> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
>> >> > collection with 4 shards ?
>> >> >
>> >> > So, how many nodes in the cluster ?
>> >> > How many shards and replicas for the collection ?
>> >> > How many items in the collection ?
>> >> > What is the size of the index ?
>> >> > How is updated the collection (frequency, how many items per days,
>> what
>> >> is
>> >> > your hard commit strategy) ?
>> >> > How are configured cache in solrconfig.xml ?
>> >> > Can you provide all other JVM parameters ?
>> >> >
>> >> > Regards
>> >> >
>> >> > Dominique
>> >> >
>> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
>> >> >
>> >> > > Second most important part of your message:
>> >> > > "When executing a huge query with many wildcards inside it the
>> server"
>> >> > >
>> >> > > This is usually an anti-pattern. The very first thing
>> >> > > I'd be doing is trying to not do this. See ngrams for infix
>> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
>> >> > >
>> >> > > And if your corpus is very large with many unique terms it's even
>> >> > > worse, but you haven't really told us about that yet.
>> >> > >
>> >> > > Best,
>> >> > > Erick
>> >> > >
>> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
>> >> > wrote:
>> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
>> >> > > >> Hi,
>> >> > > >>
>> >> > > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on
>> >> one
>> >> > of
>> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and
>> >> -Xms
>> >> > > is
>> >> > > >> 24g.
>> >> > > >> When executing a huge query with many wildcards inside it the
>> >> server
>> >> > > >> crashes and becomes non-responsive. Even the dashboard does not
>> >> > responds
>> >> > > >> and shows connection lost error. This requires me to restart the
>> >> > > servers.
>> >> > > >
>> >> > > > Here's the important part of your message:
>> >> > > >
>> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
>> >> > > >
>> >> > > >
>> >> > > > Your heap is not big enough for what Solr has been asked to do.
>> You
>> >> > > > need to either increase your heap size or change your
>> configuration
>> >> so
>> >> > > > that it uses less memory.
>> >> > > >
>> >> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>> >> > > >
>> >> > > > Most programs have pretty much undefined behavior when an OOME
>> >> occurs.
>> >> > > > Lucene's IndexWriter has been hardened so that it tries extremely
>> >> hard
>> >> > > > to avoid index corruption when OOME strikes, and I believe that
>> >> works
>> >> > > > well enough that we can call it nearly bulletproof ... but the
>> rest
>> >> of
>> >> > > > Lucene and Solr will make no guarantees.
>> >> > > >
>> >> > > > It's very difficult to have definable program behavior when an
>> OOME
>> >> > > > happens, because you simply cannot know the precise point during
>> >> > program
>> >> > > > execution where it will happen, or what isn't going to work
>> because
>> >> > Java
>> >> > > > did not have memory space to create an object.  Going unresponsive
>> >> is
>> >> > > > not surprising.
>> >> > > >
>> >> > > > If you can solve your heap problem, note that you may run into
>> other
>> >> > > > performance issues discussed on the wiki page that I linked.
>> >> > > >
>> >> > > > Thanks,
>> >> > > > Shawn
>> >> > > >
>> >> > >
>> >> >
>> >>
>> >>
>> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
>> >> dominique.bejean@eolya.fr
>> >> > wrote:
>> >>
>> >> > Hi,
>> >> >
>> >> > I agree Erick it could be a good think to have more details about your
>> >> > configuration and collection.
>> >> >
>> >> > Your heap size is 32Gb. How many RAM on each servers ?
>> >> >
>> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
>> >> > collection with 4 shards ?
>> >> >
>> >> > So, how many nodes in the cluster ?
>> >> > How many shards and replicas for the collection ?
>> >> > How many items in the collection ?
>> >> > What is the size of the index ?
>> >> > How is updated the collection (frequency, how many items per days,
>> what
>> >> is
>> >> > your hard commit strategy) ?
>> >> > How are configured cache in solrconfig.xml ?
>> >> > Can you provide all other JVM parameters ?
>> >> >
>> >> > Regards
>> >> >
>> >> > Dominique
>> >> >
>> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
>> >> >
>> >> > > Second most important part of your message:
>> >> > > "When executing a huge query with many wildcards inside it the
>> server"
>> >> > >
>> >> > > This is usually an anti-pattern. The very first thing
>> >> > > I'd be doing is trying to not do this. See ngrams for infix
>> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
>> >> > >
>> >> > > And if your corpus is very large with many unique terms it's even
>> >> > > worse, but you haven't really told us about that yet.
>> >> > >
>> >> > > Best,
>> >> > > Erick
>> >> > >
>> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
>> >> > wrote:
>> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
>> >> > > >> Hi,
>> >> > > >>
>> >> > > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on
>> >> one
>> >> > of
>> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and
>> >> -Xms
>> >> > > is
>> >> > > >> 24g.
>> >> > > >> When executing a huge query with many wildcards inside it the
>> >> server
>> >> > > >> crashes and becomes non-responsive. Even the dashboard does not
>> >> > responds
>> >> > > >> and shows connection lost error. This requires me to restart the
>> >> > > servers.
>> >> > > >
>> >> > > > Here's the important part of your message:
>> >> > > >
>> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
>> >> > > >
>> >> > > >
>> >> > > > Your heap is not big enough for what Solr has been asked to do.
>> You
>> >> > > > need to either increase your heap size or change your
>> configuration
>> >> so
>> >> > > > that it uses less memory.
>> >> > > >
>> >> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>> >> > > >
>> >> > > > Most programs have pretty much undefined behavior when an OOME
>> >> occurs.
>> >> > > > Lucene's IndexWriter has been hardened so that it tries extremely
>> >> hard
>> >> > > > to avoid index corruption when OOME strikes, and I believe that
>> >> works
>> >> > > > well enough that we can call it nearly bulletproof ... but the
>> rest
>> >> of
>> >> > > > Lucene and Solr will make no guarantees.
>> >> > > >
>> >> > > > It's very difficult to have definable program behavior when an
>> OOME
>> >> > > > happens, because you simply cannot know the precise point during
>> >> > program
>> >> > > > execution where it will happen, or what isn't going to work
>> because
>> >> > Java
>> >> > > > did not have memory space to create an object.  Going unresponsive
>> >> is
>> >> > > > not surprising.
>> >> > > >
>> >> > > > If you can solve your heap problem, note that you may run into
>> other
>> >> > > > performance issues discussed on the wiki page that I linked.
>> >> > > >
>> >> > > > Thanks,
>> >> > > > Shawn
>> >> > > >
>> >> > >
>> >> >
>> >>
>> >
>> >
>>

Re: Solr server becomes non-responsive.

Posted by Modassar Ather <mo...@gmail.com>.

Thanks for your response.

How many items in the collection ?
There are about 100 millions documents.

How are configured cache in solrconfig.xml ?
Each cache has size attribute as 128.

Can you provide a sample of the query ?
Does it fail immediately after solrcloud startup or after several hours ?
It is a query with many terms(more than a thousand) and phrase where
phrases have many wildcards in it.
Once such query is executed there are many zookeeper related exceptions and
with a couple of such queries executed it goes for OutOfMemory.

Thanks,
Modassar


On Wed, Dec 24, 2014 at 1:37 PM, Dominique Bejean <dominique.bejean@eolya.fr
> wrote:

> And you didn’t give how many RAM on each servers ?
>
> 2014-12-24 8:17 GMT+01:00 Dominique Bejean <do...@eolya.fr>:
>
> > Modassar,
> >
> > How many items in the collection ?
> > I mean how many documents per collection ? 1 million, 10 millions, …?
> >
> > How are configured cache in solrconfig.xml ?
> > What are the size attribute value for each cache ?
> >
> > Can you provide a sample of the query ?
> > Does it fail immediately after solrcloud startup or after several hours ?
> >
> > Dominique
> >
> > 2014-12-24 6:20 GMT+01:00 Modassar Ather <mo...@gmail.com>:
> >
> >> Thanks for your suggestions.
> >>
> >> I will look into the link provided.
> >> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >>
> >> This is usually an anti-pattern. The very first thing
> >> I'd be doing is trying to not do this. See ngrams for infix
> >> queries, or shingles or ReverseWildcardFilterFactory or.....
> >>
> >> We cannot avoid multiple wildcards since that's is our user's
> requirement.
> >> We try to discourage it but the users insist on firing such queries.
> Also,
> >> ngrams etc. can be tried but our index is already huge and ngrams may
> >> further add lot to it. We are OK with such queries failing as long as
> >> other
> >> queries are not affected.
> >>
> >>
> >> Please find the details below.
> >>
> >> So, how many nodes in the cluster ?
> >> There are total 4 nodes on the cluster.
> >>
> >> How many shards and replicas for the collection ?
> >> There are 4 shards and no replica for any of them.
> >>
> >> How many items in the collection ?
> >> If I understand the question correctly there are two collection on each
> >> node and there size on each node is approximately 190GB and 130GB.
> >>
> >> What is the size of the index ?
> >> There are two collection on each node and there size on each node is
> >> approximately 190GB and 130GB.
> >>
> >> How is updated the collection (frequency, how many items per days, what
> is
> >> your hard commit strategy) ?
> >> It is an optimized index and read-only. There are no inter-mediate
> update.
> >>
> >> How are configured cache in solrconfig.xml ?
> >> Filter cache, query result cache and document cache are enabled.
> >> Auto-warming is also done.
> >>
> >> Can you provide all other JVM parameters ?
> >> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
> >>
> >> Thanks again,
> >> Modassar
> >>
> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> >> dominique.bejean@eolya.fr
> >> > wrote:
> >>
> >> > Hi,
> >> >
> >> > I agree Erick it could be a good think to have more details about your
> >> > configuration and collection.
> >> >
> >> > Your heap size is 32Gb. How many RAM on each servers ?
> >> >
> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> >> > collection with 4 shards ?
> >> >
> >> > So, how many nodes in the cluster ?
> >> > How many shards and replicas for the collection ?
> >> > How many items in the collection ?
> >> > What is the size of the index ?
> >> > How is updated the collection (frequency, how many items per days,
> what
> >> is
> >> > your hard commit strategy) ?
> >> > How are configured cache in solrconfig.xml ?
> >> > Can you provide all other JVM parameters ?
> >> >
> >> > Regards
> >> >
> >> > Dominique
> >> >
> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
> >> >
> >> > > Second most important part of your message:
> >> > > "When executing a huge query with many wildcards inside it the
> server"
> >> > >
> >> > > This is usually an anti-pattern. The very first thing
> >> > > I'd be doing is trying to not do this. See ngrams for infix
> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> >> > >
> >> > > And if your corpus is very large with many unique terms it's even
> >> > > worse, but you haven't really told us about that yet.
> >> > >
> >> > > Best,
> >> > > Erick
> >> > >
> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
> >> > wrote:
> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> >> > > >> Hi,
> >> > > >>
> >> > > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on
> >> one
> >> > of
> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and
> >> -Xms
> >> > > is
> >> > > >> 24g.
> >> > > >> When executing a huge query with many wildcards inside it the
> >> server
> >> > > >> crashes and becomes non-responsive. Even the dashboard does not
> >> > responds
> >> > > >> and shows connection lost error. This requires me to restart the
> >> > > servers.
> >> > > >
> >> > > > Here's the important part of your message:
> >> > > >
> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> >> > > >
> >> > > >
> >> > > > Your heap is not big enough for what Solr has been asked to do.
> You
> >> > > > need to either increase your heap size or change your
> configuration
> >> so
> >> > > > that it uses less memory.
> >> > > >
> >> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >> > > >
> >> > > > Most programs have pretty much undefined behavior when an OOME
> >> occurs.
> >> > > > Lucene's IndexWriter has been hardened so that it tries extremely
> >> hard
> >> > > > to avoid index corruption when OOME strikes, and I believe that
> >> works
> >> > > > well enough that we can call it nearly bulletproof ... but the
> rest
> >> of
> >> > > > Lucene and Solr will make no guarantees.
> >> > > >
> >> > > > It's very difficult to have definable program behavior when an
> OOME
> >> > > > happens, because you simply cannot know the precise point during
> >> > program
> >> > > > execution where it will happen, or what isn't going to work
> because
> >> > Java
> >> > > > did not have memory space to create an object.  Going unresponsive
> >> is
> >> > > > not surprising.
> >> > > >
> >> > > > If you can solve your heap problem, note that you may run into
> other
> >> > > > performance issues discussed on the wiki page that I linked.
> >> > > >
> >> > > > Thanks,
> >> > > > Shawn
> >> > > >
> >> > >
> >> >
> >>
> >>
> >> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> >> dominique.bejean@eolya.fr
> >> > wrote:
> >>
> >> > Hi,
> >> >
> >> > I agree Erick it could be a good think to have more details about your
> >> > configuration and collection.
> >> >
> >> > Your heap size is 32Gb. How many RAM on each servers ?
> >> >
> >> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> >> > collection with 4 shards ?
> >> >
> >> > So, how many nodes in the cluster ?
> >> > How many shards and replicas for the collection ?
> >> > How many items in the collection ?
> >> > What is the size of the index ?
> >> > How is updated the collection (frequency, how many items per days,
> what
> >> is
> >> > your hard commit strategy) ?
> >> > How are configured cache in solrconfig.xml ?
> >> > Can you provide all other JVM parameters ?
> >> >
> >> > Regards
> >> >
> >> > Dominique
> >> >
> >> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
> >> >
> >> > > Second most important part of your message:
> >> > > "When executing a huge query with many wildcards inside it the
> server"
> >> > >
> >> > > This is usually an anti-pattern. The very first thing
> >> > > I'd be doing is trying to not do this. See ngrams for infix
> >> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> >> > >
> >> > > And if your corpus is very large with many unique terms it's even
> >> > > worse, but you haven't really told us about that yet.
> >> > >
> >> > > Best,
> >> > > Erick
> >> > >
> >> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
> >> > wrote:
> >> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> >> > > >> Hi,
> >> > > >>
> >> > > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on
> >> one
> >> > of
> >> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and
> >> -Xms
> >> > > is
> >> > > >> 24g.
> >> > > >> When executing a huge query with many wildcards inside it the
> >> server
> >> > > >> crashes and becomes non-responsive. Even the dashboard does not
> >> > responds
> >> > > >> and shows connection lost error. This requires me to restart the
> >> > > servers.
> >> > > >
> >> > > > Here's the important part of your message:
> >> > > >
> >> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> >> > > >
> >> > > >
> >> > > > Your heap is not big enough for what Solr has been asked to do.
> You
> >> > > > need to either increase your heap size or change your
> configuration
> >> so
> >> > > > that it uses less memory.
> >> > > >
> >> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >> > > >
> >> > > > Most programs have pretty much undefined behavior when an OOME
> >> occurs.
> >> > > > Lucene's IndexWriter has been hardened so that it tries extremely
> >> hard
> >> > > > to avoid index corruption when OOME strikes, and I believe that
> >> works
> >> > > > well enough that we can call it nearly bulletproof ... but the
> rest
> >> of
> >> > > > Lucene and Solr will make no guarantees.
> >> > > >
> >> > > > It's very difficult to have definable program behavior when an
> OOME
> >> > > > happens, because you simply cannot know the precise point during
> >> > program
> >> > > > execution where it will happen, or what isn't going to work
> because
> >> > Java
> >> > > > did not have memory space to create an object.  Going unresponsive
> >> is
> >> > > > not surprising.
> >> > > >
> >> > > > If you can solve your heap problem, note that you may run into
> other
> >> > > > performance issues discussed on the wiki page that I linked.
> >> > > >
> >> > > > Thanks,
> >> > > > Shawn
> >> > > >
> >> > >
> >> >
> >>
> >
> >
>

Re: Solr server becomes non-responsive.

Posted by Dominique Bejean <do...@eolya.fr>.

And you didn’t give how many RAM on each servers ?

2014-12-24 8:17 GMT+01:00 Dominique Bejean <do...@eolya.fr>:

> Modassar,
>
> How many items in the collection ?
> I mean how many documents per collection ? 1 million, 10 millions, …?
>
> How are configured cache in solrconfig.xml ?
> What are the size attribute value for each cache ?
>
> Can you provide a sample of the query ?
> Does it fail immediately after solrcloud startup or after several hours ?
>
> Dominique
>
> 2014-12-24 6:20 GMT+01:00 Modassar Ather <mo...@gmail.com>:
>
>> Thanks for your suggestions.
>>
>> I will look into the link provided.
>> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>>
>> This is usually an anti-pattern. The very first thing
>> I'd be doing is trying to not do this. See ngrams for infix
>> queries, or shingles or ReverseWildcardFilterFactory or.....
>>
>> We cannot avoid multiple wildcards since that's is our user's requirement.
>> We try to discourage it but the users insist on firing such queries. Also,
>> ngrams etc. can be tried but our index is already huge and ngrams may
>> further add lot to it. We are OK with such queries failing as long as
>> other
>> queries are not affected.
>>
>>
>> Please find the details below.
>>
>> So, how many nodes in the cluster ?
>> There are total 4 nodes on the cluster.
>>
>> How many shards and replicas for the collection ?
>> There are 4 shards and no replica for any of them.
>>
>> How many items in the collection ?
>> If I understand the question correctly there are two collection on each
>> node and there size on each node is approximately 190GB and 130GB.
>>
>> What is the size of the index ?
>> There are two collection on each node and there size on each node is
>> approximately 190GB and 130GB.
>>
>> How is updated the collection (frequency, how many items per days, what is
>> your hard commit strategy) ?
>> It is an optimized index and read-only. There are no inter-mediate update.
>>
>> How are configured cache in solrconfig.xml ?
>> Filter cache, query result cache and document cache are enabled.
>> Auto-warming is also done.
>>
>> Can you provide all other JVM parameters ?
>> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
>>
>> Thanks again,
>> Modassar
>>
>> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
>> dominique.bejean@eolya.fr
>> > wrote:
>>
>> > Hi,
>> >
>> > I agree Erick it could be a good think to have more details about your
>> > configuration and collection.
>> >
>> > Your heap size is 32Gb. How many RAM on each servers ?
>> >
>> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
>> > collection with 4 shards ?
>> >
>> > So, how many nodes in the cluster ?
>> > How many shards and replicas for the collection ?
>> > How many items in the collection ?
>> > What is the size of the index ?
>> > How is updated the collection (frequency, how many items per days, what
>> is
>> > your hard commit strategy) ?
>> > How are configured cache in solrconfig.xml ?
>> > Can you provide all other JVM parameters ?
>> >
>> > Regards
>> >
>> > Dominique
>> >
>> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
>> >
>> > > Second most important part of your message:
>> > > "When executing a huge query with many wildcards inside it the server"
>> > >
>> > > This is usually an anti-pattern. The very first thing
>> > > I'd be doing is trying to not do this. See ngrams for infix
>> > > queries, or shingles or ReverseWildcardFilterFactory or.....
>> > >
>> > > And if your corpus is very large with many unique terms it's even
>> > > worse, but you haven't really told us about that yet.
>> > >
>> > > Best,
>> > > Erick
>> > >
>> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
>> > wrote:
>> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
>> > > >> Hi,
>> > > >>
>> > > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on
>> one
>> > of
>> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and
>> -Xms
>> > > is
>> > > >> 24g.
>> > > >> When executing a huge query with many wildcards inside it the
>> server
>> > > >> crashes and becomes non-responsive. Even the dashboard does not
>> > responds
>> > > >> and shows connection lost error. This requires me to restart the
>> > > servers.
>> > > >
>> > > > Here's the important part of your message:
>> > > >
>> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
>> > > >
>> > > >
>> > > > Your heap is not big enough for what Solr has been asked to do.  You
>> > > > need to either increase your heap size or change your configuration
>> so
>> > > > that it uses less memory.
>> > > >
>> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>> > > >
>> > > > Most programs have pretty much undefined behavior when an OOME
>> occurs.
>> > > > Lucene's IndexWriter has been hardened so that it tries extremely
>> hard
>> > > > to avoid index corruption when OOME strikes, and I believe that
>> works
>> > > > well enough that we can call it nearly bulletproof ... but the rest
>> of
>> > > > Lucene and Solr will make no guarantees.
>> > > >
>> > > > It's very difficult to have definable program behavior when an OOME
>> > > > happens, because you simply cannot know the precise point during
>> > program
>> > > > execution where it will happen, or what isn't going to work because
>> > Java
>> > > > did not have memory space to create an object.  Going unresponsive
>> is
>> > > > not surprising.
>> > > >
>> > > > If you can solve your heap problem, note that you may run into other
>> > > > performance issues discussed on the wiki page that I linked.
>> > > >
>> > > > Thanks,
>> > > > Shawn
>> > > >
>> > >
>> >
>>
>>
>> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
>> dominique.bejean@eolya.fr
>> > wrote:
>>
>> > Hi,
>> >
>> > I agree Erick it could be a good think to have more details about your
>> > configuration and collection.
>> >
>> > Your heap size is 32Gb. How many RAM on each servers ?
>> >
>> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
>> > collection with 4 shards ?
>> >
>> > So, how many nodes in the cluster ?
>> > How many shards and replicas for the collection ?
>> > How many items in the collection ?
>> > What is the size of the index ?
>> > How is updated the collection (frequency, how many items per days, what
>> is
>> > your hard commit strategy) ?
>> > How are configured cache in solrconfig.xml ?
>> > Can you provide all other JVM parameters ?
>> >
>> > Regards
>> >
>> > Dominique
>> >
>> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
>> >
>> > > Second most important part of your message:
>> > > "When executing a huge query with many wildcards inside it the server"
>> > >
>> > > This is usually an anti-pattern. The very first thing
>> > > I'd be doing is trying to not do this. See ngrams for infix
>> > > queries, or shingles or ReverseWildcardFilterFactory or.....
>> > >
>> > > And if your corpus is very large with many unique terms it's even
>> > > worse, but you haven't really told us about that yet.
>> > >
>> > > Best,
>> > > Erick
>> > >
>> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
>> > wrote:
>> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
>> > > >> Hi,
>> > > >>
>> > > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on
>> one
>> > of
>> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and
>> -Xms
>> > > is
>> > > >> 24g.
>> > > >> When executing a huge query with many wildcards inside it the
>> server
>> > > >> crashes and becomes non-responsive. Even the dashboard does not
>> > responds
>> > > >> and shows connection lost error. This requires me to restart the
>> > > servers.
>> > > >
>> > > > Here's the important part of your message:
>> > > >
>> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
>> > > >
>> > > >
>> > > > Your heap is not big enough for what Solr has been asked to do.  You
>> > > > need to either increase your heap size or change your configuration
>> so
>> > > > that it uses less memory.
>> > > >
>> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>> > > >
>> > > > Most programs have pretty much undefined behavior when an OOME
>> occurs.
>> > > > Lucene's IndexWriter has been hardened so that it tries extremely
>> hard
>> > > > to avoid index corruption when OOME strikes, and I believe that
>> works
>> > > > well enough that we can call it nearly bulletproof ... but the rest
>> of
>> > > > Lucene and Solr will make no guarantees.
>> > > >
>> > > > It's very difficult to have definable program behavior when an OOME
>> > > > happens, because you simply cannot know the precise point during
>> > program
>> > > > execution where it will happen, or what isn't going to work because
>> > Java
>> > > > did not have memory space to create an object.  Going unresponsive
>> is
>> > > > not surprising.
>> > > >
>> > > > If you can solve your heap problem, note that you may run into other
>> > > > performance issues discussed on the wiki page that I linked.
>> > > >
>> > > > Thanks,
>> > > > Shawn
>> > > >
>> > >
>> >
>>
>
>

Re: Solr server becomes non-responsive.

Posted by Dominique Bejean <do...@eolya.fr>.

Modassar,

How many items in the collection ?
I mean how many documents per collection ? 1 million, 10 millions, …?

How are configured cache in solrconfig.xml ?
What are the size attribute value for each cache ?

Can you provide a sample of the query ?
Does it fail immediately after solrcloud startup or after several hours ?

Dominique

2014-12-24 6:20 GMT+01:00 Modassar Ather <mo...@gmail.com>:

> Thanks for your suggestions.
>
> I will look into the link provided.
> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>
> This is usually an anti-pattern. The very first thing
> I'd be doing is trying to not do this. See ngrams for infix
> queries, or shingles or ReverseWildcardFilterFactory or.....
>
> We cannot avoid multiple wildcards since that's is our user's requirement.
> We try to discourage it but the users insist on firing such queries. Also,
> ngrams etc. can be tried but our index is already huge and ngrams may
> further add lot to it. We are OK with such queries failing as long as other
> queries are not affected.
>
>
> Please find the details below.
>
> So, how many nodes in the cluster ?
> There are total 4 nodes on the cluster.
>
> How many shards and replicas for the collection ?
> There are 4 shards and no replica for any of them.
>
> How many items in the collection ?
> If I understand the question correctly there are two collection on each
> node and there size on each node is approximately 190GB and 130GB.
>
> What is the size of the index ?
> There are two collection on each node and there size on each node is
> approximately 190GB and 130GB.
>
> How is updated the collection (frequency, how many items per days, what is
> your hard commit strategy) ?
> It is an optimized index and read-only. There are no inter-mediate update.
>
> How are configured cache in solrconfig.xml ?
> Filter cache, query result cache and document cache are enabled.
> Auto-warming is also done.
>
> Can you provide all other JVM parameters ?
> -Xms20g -Xmx24g -XX:+UseConcMarkSweepGC
>
> Thanks again,
> Modassar
>
> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> dominique.bejean@eolya.fr
> > wrote:
>
> > Hi,
> >
> > I agree Erick it could be a good think to have more details about your
> > configuration and collection.
> >
> > Your heap size is 32Gb. How many RAM on each servers ?
> >
> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> > collection with 4 shards ?
> >
> > So, how many nodes in the cluster ?
> > How many shards and replicas for the collection ?
> > How many items in the collection ?
> > What is the size of the index ?
> > How is updated the collection (frequency, how many items per days, what
> is
> > your hard commit strategy) ?
> > How are configured cache in solrconfig.xml ?
> > Can you provide all other JVM parameters ?
> >
> > Regards
> >
> > Dominique
> >
> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
> >
> > > Second most important part of your message:
> > > "When executing a huge query with many wildcards inside it the server"
> > >
> > > This is usually an anti-pattern. The very first thing
> > > I'd be doing is trying to not do this. See ngrams for infix
> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> > >
> > > And if your corpus is very large with many unique terms it's even
> > > worse, but you haven't really told us about that yet.
> > >
> > > Best,
> > > Erick
> > >
> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
> > wrote:
> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > > >> Hi,
> > > >>
> > > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on
> one
> > of
> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and
> -Xms
> > > is
> > > >> 24g.
> > > >> When executing a huge query with many wildcards inside it the server
> > > >> crashes and becomes non-responsive. Even the dashboard does not
> > responds
> > > >> and shows connection lost error. This requires me to restart the
> > > servers.
> > > >
> > > > Here's the important part of your message:
> > > >
> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> > > >
> > > >
> > > > Your heap is not big enough for what Solr has been asked to do.  You
> > > > need to either increase your heap size or change your configuration
> so
> > > > that it uses less memory.
> > > >
> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > > >
> > > > Most programs have pretty much undefined behavior when an OOME
> occurs.
> > > > Lucene's IndexWriter has been hardened so that it tries extremely
> hard
> > > > to avoid index corruption when OOME strikes, and I believe that works
> > > > well enough that we can call it nearly bulletproof ... but the rest
> of
> > > > Lucene and Solr will make no guarantees.
> > > >
> > > > It's very difficult to have definable program behavior when an OOME
> > > > happens, because you simply cannot know the precise point during
> > program
> > > > execution where it will happen, or what isn't going to work because
> > Java
> > > > did not have memory space to create an object.  Going unresponsive is
> > > > not surprising.
> > > >
> > > > If you can solve your heap problem, note that you may run into other
> > > > performance issues discussed on the wiki page that I linked.
> > > >
> > > > Thanks,
> > > > Shawn
> > > >
> > >
> >
>
>
> On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <
> dominique.bejean@eolya.fr
> > wrote:
>
> > Hi,
> >
> > I agree Erick it could be a good think to have more details about your
> > configuration and collection.
> >
> > Your heap size is 32Gb. How many RAM on each servers ?
> >
> > By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> > collection with 4 shards ?
> >
> > So, how many nodes in the cluster ?
> > How many shards and replicas for the collection ?
> > How many items in the collection ?
> > What is the size of the index ?
> > How is updated the collection (frequency, how many items per days, what
> is
> > your hard commit strategy) ?
> > How are configured cache in solrconfig.xml ?
> > Can you provide all other JVM parameters ?
> >
> > Regards
> >
> > Dominique
> >
> > 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
> >
> > > Second most important part of your message:
> > > "When executing a huge query with many wildcards inside it the server"
> > >
> > > This is usually an anti-pattern. The very first thing
> > > I'd be doing is trying to not do this. See ngrams for infix
> > > queries, or shingles or ReverseWildcardFilterFactory or.....
> > >
> > > And if your corpus is very large with many unique terms it's even
> > > worse, but you haven't really told us about that yet.
> > >
> > > Best,
> > > Erick
> > >
> > > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
> > wrote:
> > > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > > >> Hi,
> > > >>
> > > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on
> one
> > of
> > > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and
> -Xms
> > > is
> > > >> 24g.
> > > >> When executing a huge query with many wildcards inside it the server
> > > >> crashes and becomes non-responsive. Even the dashboard does not
> > responds
> > > >> and shows connection lost error. This requires me to restart the
> > > servers.
> > > >
> > > > Here's the important part of your message:
> > > >
> > > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> > > >
> > > >
> > > > Your heap is not big enough for what Solr has been asked to do.  You
> > > > need to either increase your heap size or change your configuration
> so
> > > > that it uses less memory.
> > > >
> > > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > > >
> > > > Most programs have pretty much undefined behavior when an OOME
> occurs.
> > > > Lucene's IndexWriter has been hardened so that it tries extremely
> hard
> > > > to avoid index corruption when OOME strikes, and I believe that works
> > > > well enough that we can call it nearly bulletproof ... but the rest
> of
> > > > Lucene and Solr will make no guarantees.
> > > >
> > > > It's very difficult to have definable program behavior when an OOME
> > > > happens, because you simply cannot know the precise point during
> > program
> > > > execution where it will happen, or what isn't going to work because
> > Java
> > > > did not have memory space to create an object.  Going unresponsive is
> > > > not surprising.
> > > >
> > > > If you can solve your heap problem, note that you may run into other
> > > > performance issues discussed on the wiki page that I linked.
> > > >
> > > > Thanks,
> > > > Shawn
> > > >
> > >
> >
>

Re: Solr server becomes non-responsive.

Posted by Modassar Ather <mo...@gmail.com>.

Thanks for your suggestions.

I will look into the link provided.
http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap

This is usually an anti-pattern. The very first thing
I'd be doing is trying to not do this. See ngrams for infix
queries, or shingles or ReverseWildcardFilterFactory or.....

We cannot avoid multiple wildcards since that's is our user's requirement.
We try to discourage it but the users insist on firing such queries. Also,
ngrams etc. can be tried but our index is already huge and ngrams may
further add lot to it. We are OK with such queries failing as long as other
queries are not affected.


Please find the details below.

So, how many nodes in the cluster ?
There are total 4 nodes on the cluster.

How many shards and replicas for the collection ?
There are 4 shards and no replica for any of them.

How many items in the collection ?
If I understand the question correctly there are two collection on each
node and there size on each node is approximately 190GB and 130GB.

What is the size of the index ?
There are two collection on each node and there size on each node is
approximately 190GB and 130GB.

How is updated the collection (frequency, how many items per days, what is
your hard commit strategy) ?
It is an optimized index and read-only. There are no inter-mediate update.

How are configured cache in solrconfig.xml ?
Filter cache, query result cache and document cache are enabled.
Auto-warming is also done.

Can you provide all other JVM parameters ?
-Xms20g -Xmx24g -XX:+UseConcMarkSweepGC

Thanks again,
Modassar

On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <dominique.bejean@eolya.fr
> wrote:

> Hi,
>
> I agree Erick it could be a good think to have more details about your
> configuration and collection.
>
> Your heap size is 32Gb. How many RAM on each servers ?
>
> By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> collection with 4 shards ?
>
> So, how many nodes in the cluster ?
> How many shards and replicas for the collection ?
> How many items in the collection ?
> What is the size of the index ?
> How is updated the collection (frequency, how many items per days, what is
> your hard commit strategy) ?
> How are configured cache in solrconfig.xml ?
> Can you provide all other JVM parameters ?
>
> Regards
>
> Dominique
>
> 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
>
> > Second most important part of your message:
> > "When executing a huge query with many wildcards inside it the server"
> >
> > This is usually an anti-pattern. The very first thing
> > I'd be doing is trying to not do this. See ngrams for infix
> > queries, or shingles or ReverseWildcardFilterFactory or.....
> >
> > And if your corpus is very large with many unique terms it's even
> > worse, but you haven't really told us about that yet.
> >
> > Best,
> > Erick
> >
> > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
> wrote:
> > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > >> Hi,
> > >>
> > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on one
> of
> > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and -Xms
> > is
> > >> 24g.
> > >> When executing a huge query with many wildcards inside it the server
> > >> crashes and becomes non-responsive. Even the dashboard does not
> responds
> > >> and shows connection lost error. This requires me to restart the
> > servers.
> > >
> > > Here's the important part of your message:
> > >
> > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> > >
> > >
> > > Your heap is not big enough for what Solr has been asked to do.  You
> > > need to either increase your heap size or change your configuration so
> > > that it uses less memory.
> > >
> > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > >
> > > Most programs have pretty much undefined behavior when an OOME occurs.
> > > Lucene's IndexWriter has been hardened so that it tries extremely hard
> > > to avoid index corruption when OOME strikes, and I believe that works
> > > well enough that we can call it nearly bulletproof ... but the rest of
> > > Lucene and Solr will make no guarantees.
> > >
> > > It's very difficult to have definable program behavior when an OOME
> > > happens, because you simply cannot know the precise point during
> program
> > > execution where it will happen, or what isn't going to work because
> Java
> > > did not have memory space to create an object.  Going unresponsive is
> > > not surprising.
> > >
> > > If you can solve your heap problem, note that you may run into other
> > > performance issues discussed on the wiki page that I linked.
> > >
> > > Thanks,
> > > Shawn
> > >
> >
>


On Wed, Dec 24, 2014 at 2:29 AM, Dominique Bejean <dominique.bejean@eolya.fr
> wrote:

> Hi,
>
> I agree Erick it could be a good think to have more details about your
> configuration and collection.
>
> Your heap size is 32Gb. How many RAM on each servers ?
>
> By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
> collection with 4 shards ?
>
> So, how many nodes in the cluster ?
> How many shards and replicas for the collection ?
> How many items in the collection ?
> What is the size of the index ?
> How is updated the collection (frequency, how many items per days, what is
> your hard commit strategy) ?
> How are configured cache in solrconfig.xml ?
> Can you provide all other JVM parameters ?
>
> Regards
>
> Dominique
>
> 2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:
>
> > Second most important part of your message:
> > "When executing a huge query with many wildcards inside it the server"
> >
> > This is usually an anti-pattern. The very first thing
> > I'd be doing is trying to not do this. See ngrams for infix
> > queries, or shingles or ReverseWildcardFilterFactory or.....
> >
> > And if your corpus is very large with many unique terms it's even
> > worse, but you haven't really told us about that yet.
> >
> > Best,
> > Erick
> >
> > On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org>
> wrote:
> > > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> > >> Hi,
> > >>
> > >> I have a setup of 4 shard Solr cluster with embedded zookeeper on one
> of
> > >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and -Xms
> > is
> > >> 24g.
> > >> When executing a huge query with many wildcards inside it the server
> > >> crashes and becomes non-responsive. Even the dashboard does not
> responds
> > >> and shows connection lost error. This requires me to restart the
> > servers.
> > >
> > > Here's the important part of your message:
> > >
> > > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> > >
> > >
> > > Your heap is not big enough for what Solr has been asked to do.  You
> > > need to either increase your heap size or change your configuration so
> > > that it uses less memory.
> > >
> > > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> > >
> > > Most programs have pretty much undefined behavior when an OOME occurs.
> > > Lucene's IndexWriter has been hardened so that it tries extremely hard
> > > to avoid index corruption when OOME strikes, and I believe that works
> > > well enough that we can call it nearly bulletproof ... but the rest of
> > > Lucene and Solr will make no guarantees.
> > >
> > > It's very difficult to have definable program behavior when an OOME
> > > happens, because you simply cannot know the precise point during
> program
> > > execution where it will happen, or what isn't going to work because
> Java
> > > did not have memory space to create an object.  Going unresponsive is
> > > not surprising.
> > >
> > > If you can solve your heap problem, note that you may run into other
> > > performance issues discussed on the wiki page that I linked.
> > >
> > > Thanks,
> > > Shawn
> > >
> >
>

Re: Solr server becomes non-responsive.

Posted by Dominique Bejean <do...@eolya.fr>.

Hi,

I agree Erick it could be a good think to have more details about your
configuration and collection.

Your heap size is 32Gb. How many RAM on each servers ?

By « 4 shard Solr cluster », you mean a 4 nodes Solr servers or a
collection with 4 shards ?

So, how many nodes in the cluster ?
How many shards and replicas for the collection ?
How many items in the collection ?
What is the size of the index ?
How is updated the collection (frequency, how many items per days, what is
your hard commit strategy) ?
How are configured cache in solrconfig.xml ?
Can you provide all other JVM parameters ?

Regards

Dominique

2014-12-23 17:50 GMT+01:00 Erick Erickson <er...@gmail.com>:

> Second most important part of your message:
> "When executing a huge query with many wildcards inside it the server"
>
> This is usually an anti-pattern. The very first thing
> I'd be doing is trying to not do this. See ngrams for infix
> queries, or shingles or ReverseWildcardFilterFactory or.....
>
> And if your corpus is very large with many unique terms it's even
> worse, but you haven't really told us about that yet.
>
> Best,
> Erick
>
> On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org> wrote:
> > On 12/23/2014 4:34 AM, Modassar Ather wrote:
> >> Hi,
> >>
> >> I have a setup of 4 shard Solr cluster with embedded zookeeper on one of
> >> them. The zkClient time out is set to 30 seconds, -Xms is 20g and -Xms
> is
> >> 24g.
> >> When executing a huge query with many wildcards inside it the server
> >> crashes and becomes non-responsive. Even the dashboard does not responds
> >> and shows connection lost error. This requires me to restart the
> servers.
> >
> > Here's the important part of your message:
> >
> > *Caused by: java.lang.OutOfMemoryError: Java heap space*
> >
> >
> > Your heap is not big enough for what Solr has been asked to do.  You
> > need to either increase your heap size or change your configuration so
> > that it uses less memory.
> >
> > http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
> >
> > Most programs have pretty much undefined behavior when an OOME occurs.
> > Lucene's IndexWriter has been hardened so that it tries extremely hard
> > to avoid index corruption when OOME strikes, and I believe that works
> > well enough that we can call it nearly bulletproof ... but the rest of
> > Lucene and Solr will make no guarantees.
> >
> > It's very difficult to have definable program behavior when an OOME
> > happens, because you simply cannot know the precise point during program
> > execution where it will happen, or what isn't going to work because Java
> > did not have memory space to create an object.  Going unresponsive is
> > not surprising.
> >
> > If you can solve your heap problem, note that you may run into other
> > performance issues discussed on the wiki page that I linked.
> >
> > Thanks,
> > Shawn
> >
>

Re: Solr server becomes non-responsive.

Posted by Erick Erickson <er...@gmail.com>.

Second most important part of your message:
"When executing a huge query with many wildcards inside it the server"

This is usually an anti-pattern. The very first thing
I'd be doing is trying to not do this. See ngrams for infix
queries, or shingles or ReverseWildcardFilterFactory or.....

And if your corpus is very large with many unique terms it's even
worse, but you haven't really told us about that yet.

Best,
Erick

On Tue, Dec 23, 2014 at 8:30 AM, Shawn Heisey <ap...@elyograg.org> wrote:
> On 12/23/2014 4:34 AM, Modassar Ather wrote:
>> Hi,
>>
>> I have a setup of 4 shard Solr cluster with embedded zookeeper on one of
>> them. The zkClient time out is set to 30 seconds, -Xms is 20g and -Xms is
>> 24g.
>> When executing a huge query with many wildcards inside it the server
>> crashes and becomes non-responsive. Even the dashboard does not responds
>> and shows connection lost error. This requires me to restart the servers.
>
> Here's the important part of your message:
>
> *Caused by: java.lang.OutOfMemoryError: Java heap space*
>
>
> Your heap is not big enough for what Solr has been asked to do.  You
> need to either increase your heap size or change your configuration so
> that it uses less memory.
>
> http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap
>
> Most programs have pretty much undefined behavior when an OOME occurs.
> Lucene's IndexWriter has been hardened so that it tries extremely hard
> to avoid index corruption when OOME strikes, and I believe that works
> well enough that we can call it nearly bulletproof ... but the rest of
> Lucene and Solr will make no guarantees.
>
> It's very difficult to have definable program behavior when an OOME
> happens, because you simply cannot know the precise point during program
> execution where it will happen, or what isn't going to work because Java
> did not have memory space to create an object.  Going unresponsive is
> not surprising.
>
> If you can solve your heap problem, note that you may run into other
> performance issues discussed on the wiki page that I linked.
>
> Thanks,
> Shawn
>

Re: Solr server becomes non-responsive.

Posted by Shawn Heisey <ap...@elyograg.org>.

On 12/23/2014 4:34 AM, Modassar Ather wrote:
> Hi,
>
> I have a setup of 4 shard Solr cluster with embedded zookeeper on one of
> them. The zkClient time out is set to 30 seconds, -Xms is 20g and -Xms is
> 24g.
> When executing a huge query with many wildcards inside it the server
> crashes and becomes non-responsive. Even the dashboard does not responds
> and shows connection lost error. This requires me to restart the servers.

Here's the important part of your message:

*Caused by: java.lang.OutOfMemoryError: Java heap space*

Your heap is not big enough for what Solr has been asked to do.  You
need to either increase your heap size or change your configuration so
that it uses less memory.

http://wiki.apache.org/solr/SolrPerformanceProblems#Java_Heap

Most programs have pretty much undefined behavior when an OOME occurs. 
Lucene's IndexWriter has been hardened so that it tries extremely hard
to avoid index corruption when OOME strikes, and I believe that works
well enough that we can call it nearly bulletproof ... but the rest of
Lucene and Solr will make no guarantees.

It's very difficult to have definable program behavior when an OOME
happens, because you simply cannot know the precise point during program
execution where it will happen, or what isn't going to work because Java
did not have memory space to create an object.  Going unresponsive is
not surprising.

If you can solve your heap problem, note that you may run into other
performance issues discussed on the wiki page that I linked.

Thanks,
Shawn

Re: Solr server becomes non-responsive.

Posted by Modassar Ather <mo...@gmail.com>.

To add to the details of above issue the query as soon it is executed, even
before the OutOfMemory error causes the solr servers to become
non-responsive.

On Tue, Dec 23, 2014 at 5:04 PM, Modassar Ather <mo...@gmail.com>
wrote:

> Hi,
>
> I have a setup of 4 shard Solr cluster with embedded zookeeper on one of
> them. The zkClient time out is set to 30 seconds, -Xms is 20g and -Xms is
> 24g.
> When executing a huge query with many wildcards inside it the server
> crashes and becomes non-responsive. Even the dashboard does not responds
> and shows connection lost error. This requires me to restart the servers.
>
> I have set query *timeAllowed* to 5 minutes but it also seems to be not
> getting honored and the query hangs around.
>
> Kindly help me debug the issue and fix it or if there is a way the
> timeAllowed can be made honored or a the query which is hanging for
> sometime can be stopped.
>
> *Following are few exceptions.*
>
> *org.apache.zookeeper.server.NIOServerCnxn doIO*
> WARNING: caught end of stream exception
> EndOfStreamException: Unable to read additional data from client sessionid
> <session id>, likely client has closed socket
>         at
> org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
>         at
> org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>         at java.lang.Thread.run(Thread.java:745)
>
> *org.apache.zookeeper.server.NIOServerCnxn sendBuffer*
> SEVERE: Unexpected Exception:
> java.nio.channels.CancelledKeyException
>         at
> sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
>         at
> sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:77)
>         at
> org.apache.zookeeper.server.NIOServerCnxn.sendBuffer(NIOServerCnxn.java:151)
>         at
> org.apache.zookeeper.server.NIOServerCnxn.sendResponse(NIOServerCnxn.java:1081)
>         at
> org.apache.zookeeper.server.FinalRequestProcessor.processRequest(FinalRequestProcessor.java:404)
>         at
> org.apache.zookeeper.server.SyncRequestProcessor.flush(SyncRequestProcessor.java:200)
>         at
> org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:131)
>
> *org.apache.zookeeper.server.persistence.FileTxnLog commit*
> WARNING: fsync-ing the write ahead log in SyncThread:0 took 28346ms which
> will adversely effect operation latency. See the ZooKeeper troubleshooting
> guide
> org.apache.solr.common.SolrException log
> SEVERE: org.apache.solr.common.SolrException: no servers hosting shard:
>         at
> org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:149)
>         at
> org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:119)
>         at java.util.concurrent.FutureTask.run(FutureTask.java:262)
>         at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
>         at java.util.concurrent.FutureTask.run(FutureTask.java:262)
>         at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>         at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>         at java.lang.Thread.run(Thread.java:745)
>
> *Caused by: java.lang.OutOfMemoryError: Java heap space*
>         at
> org.apache.lucene.codecs.lucene41.Lucene41PostingsReader$BlockDocsAndPositionsEnum.<init>(Lucene41PostingsReader.java:640)
>         at
> org.apache.lucene.codecs.lucene41.Lucene41PostingsReader.docsAndPositions(Lucene41PostingsReader.java:278)
>         at
> org.apache.lucene.codecs.blocktree.SegmentTermsEnum.docsAndPositions(SegmentTermsEnum.java:1011)
>         at
> org.apache.lucene.search.spans.SpanTermQuery.getSpans(SpanTermQuery.java:123)
>         at
> org.apache.lucene.search.spans.SpanOrQuery$1.initSpanQueue(SpanOrQuery.java:180)
>         at
> org.apache.lucene.search.spans.SpanOrQuery$1.next(SpanOrQuery.java:193)
>         at
> org.apache.lucene.search.spans.SpanOrQuery$1.initSpanQueue(SpanOrQuery.java:182)
>         at
> org.apache.lucene.search.spans.SpanOrQuery$1.next(SpanOrQuery.java:193)
>         at
> org.apache.lucene.search.spans.NearSpansUnordered$SpansCell.next(NearSpansUnordered.java:88)
>         at
> org.apache.lucene.search.spans.NearSpansUnordered.initList(NearSpansUnordered.java:295)
>         at
> org.apache.lucene.search.spans.NearSpansUnordered.next(NearSpansUnordered.java:164)
>         at
> org.apache.lucene.search.spans.SpanScorer.<init>(SpanScorer.java:46)
>         at
> org.apache.lucene.search.spans.SpanWeight.scorer(SpanWeight.java:88)
>         at
> org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:160)
>         at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
>         at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
>         at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
>         at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
>         at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
>         at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
>         at
> org.apache.lucene.search.BooleanQuery$BooleanWeight.scorer(BooleanQuery.java:356)
>         at
> org.apache.lucene.queries.function.BoostedQuery$BoostedWeight.scorer(BoostedQuery.java:101)
>         at org.apache.lucene.search.Weight.bulkScorer(Weight.java:131)
>         at
> org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:618)
>         at
> org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:297)
>         at
> org.apache.solr.search.grouping.CommandHandler.searchWithTimeLimiter(CommandHandler.java:219)
>         at
> org.apache.solr.search.grouping.CommandHandler.execute(CommandHandler.java:154)
>         at
> org.apache.solr.handler.component.QueryComponent.process(QueryComponent.java:362)
>         at
> org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:218)
>         at
> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
>         at org.apache.solr.core.SolrCore.execute(SolrCore.java:1967)
>         at
> org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:777)
>
> Thanks,
> Modassar
>