You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-user@lucene.apache.org by Nathan Neulinger <nn...@neulinger.org> on 2014/07/27 01:15:11 UTC
/solr/admin/ping causing exceptions in log?
Recently deployed haproxy in front of my solr instances, and seeing a large number of exceptions in the logs now...
Example below. I can pound the server with requests against /solr/admin/ping via curl, with no obvious issue, but the
haproxy checks appear to be aggravating something.
Solr 4.8.0 w/ solr cloud, 2 nodes, 3 zk, linux x86_64
It seems like when the issue occurs, I get a set of the errors all in a burst (below), never just one.
Suggestions?
-- Nathan
------------------------------------------------------------
Nathan Neulinger nneul@neulinger.org
Neulinger Consulting (573) 612-1412
2014-07-26 23:04:36,506 ERROR qtp1532385072-4864 [g.apache.solr.servlet.SolrDispatchFilter] -
null:org.eclipse.jetty.io.EofException
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:914)
at org.eclipse.jetty.http.AbstractGenerator.flush(AbstractGenerator.java:443)
at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100)
at org.eclipse.jetty.server.AbstractHttpConnection$Output.flush(AbstractHttpConnection.java:1094)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
at org.apache.solr.util.FastWriter.flush(FastWriter.java:137)
at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:763)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:431)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368)
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Connection reset
at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:118)
at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
at org.eclipse.jetty.io.ByteArrayBuffer.writeTo(ByteArrayBuffer.java:375)
at org.eclipse.jetty.io.bio.StreamEndPoint.flush(StreamEndPoint.java:164)
at org.eclipse.jetty.io.bio.StreamEndPoint.flush(StreamEndPoint.java:194)
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:838)
... 36 more
2014-07-26 23:04:36,513 ERROR qtp1532385072-4864 [g.apache.solr.servlet.SolrDispatchFilter] -
null:org.eclipse.jetty.io.EofException
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:914)
at org.eclipse.jetty.http.AbstractGenerator.flush(AbstractGenerator.java:443)
at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100)
at org.eclipse.jetty.server.AbstractHttpConnection$Output.flush(AbstractHttpConnection.java:1094)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
at org.apache.solr.util.FastWriter.flush(FastWriter.java:137)
at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:763)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:431)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368)
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Connection reset
at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:118)
at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
at org.eclipse.jetty.io.ByteArrayBuffer.writeTo(ByteArrayBuffer.java:375)
at org.eclipse.jetty.io.bio.StreamEndPoint.flush(StreamEndPoint.java:164)
at org.eclipse.jetty.io.bio.StreamEndPoint.flush(StreamEndPoint.java:194)
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:838)
... 36 more
2014-07-26 23:04:36,514 WARN qtp1532385072-4864 [org.eclipse.jetty.server.Response ] - Committed before 500
{msg=Connection reset,trace=org.eclipse.jetty.io.EofException
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:914)
at org.eclipse.jetty.http.AbstractGenerator.flush(AbstractGenerator.java:443)
at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100)
at org.eclipse.jetty.server.AbstractHttpConnection$Output.flush(AbstractHttpConnection.java:1094)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
at org.apache.solr.util.FastWriter.flush(FastWriter.java:137)
at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:763)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:431)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368)
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Connection reset
at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:118)
at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
at org.eclipse.jetty.io.ByteArrayBuffer.writeTo(ByteArrayBuffer.java:375)
at org.eclipse.jetty.io.bio.StreamEndPoint.flush(StreamEndPoint.java:164)
at org.eclipse.jetty.io.bio.StreamEndPoint.flush(StreamEndPoint.java:194)
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:838)
... 36 more
,code=500}
2014-07-26 23:04:36,517 ERROR qtp1532385072-4864 [g.apache.solr.servlet.SolrDispatchFilter] -
null:java.lang.IllegalStateException: Committed
at org.eclipse.jetty.server.Response.resetBuffer(Response.java:1144)
at org.eclipse.jetty.server.Response.sendError(Response.java:314)
at org.apache.solr.servlet.SolrDispatchFilter.sendError(SolrDispatchFilter.java:818)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:439)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368)
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
2014-07-26 23:04:36,518 WARN qtp1532385072-4864 [org.eclipse.jetty.server.Response ] - Committed before 500
{msg=Committed,trace=java.lang.IllegalStateException: Committed
at org.eclipse.jetty.server.Response.resetBuffer(Response.java:1144)
at org.eclipse.jetty.server.Response.sendError(Response.java:314)
at org.apache.solr.servlet.SolrDispatchFilter.sendError(SolrDispatchFilter.java:818)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:439)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:339)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368)
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
,code=500}
2014-07-26 23:04:36,518 WARN qtp1532385072-4864 [org.eclipse.jetty.servlet.ServletHandler] - /solr/admin/ping
java.lang.IllegalStateException: Committed
at org.eclipse.jetty.server.Response.resetBuffer(Response.java:1144)
at org.eclipse.jetty.server.Response.sendError(Response.java:314)
at org.apache.solr.servlet.SolrDispatchFilter.sendError(SolrDispatchFilter.java:818)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:439)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368)
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
Re: /solr/admin/ping causing exceptions in log?
Posted by Nathan Neulinger <nn...@neulinger.org>.
Thing is - I wouldn't expect any of the default options mentioned to change the behavior intermittently.
i.e. it's working for 95% of the health check requests, it's just the intermittent ones that seem to be cut off... I'm
inquiring with haproxy devs since it appears that at least one other person on #haproxy is seeing the same behavior.
Doesn't appear to be specific to solr.
-- Nathan
On 07/27/2014 10:44 PM, Shawn Heisey wrote:
> On 7/27/2014 7:23 PM, Nathan Neulinger wrote:
>> Unfortunately, doesn't look like this clears the symptom.
>>
>> The ping is responding almost instantly every time. I've tried setting a
>> 15 second timeout on the check, with no change in occurences of the error.
>>
>> Looking at a packet capture on the server side, there is a clear
>> distinction between working and failing/error-triggering connections.
>>
>> It looks like in a "working" case, I see two packets immediately back to
>> back (one with header, and next a continuation with content) with no ack
>> in between, followed by ack, rst+ack, rst.
>>
>> In the failing request, I see the GET request, acked, then the http/1.1
>> 200 Ok response from Solr, a single ack, and then an almost
>> instantaneous reset sent by the client.
>>
>>
>> I'm only seeing this on traffic to/from haproxy checks. If I do a simple:
>>
>> while [ true ]; do curl -s http://host:8983/solr/admin/ping; done
>>
>> from the same box, that flood runs with generally 10-20ms request times
>> and zero errors.
>
> I won't claim to understand what's going on here, but it might be a
> matter of the haproxy options. Here are the options I'm using in the
> "defaults" section of the config:
>
> defaults
> log global
> mode http
> option httplog
> option dontlognull
> option redispatch
> option abortonclose
> option http-server-close
> option http-pretend-keepalive
> retries 1
> maxconn 1024
> timeout connect 1s
> timeout client 5s
> timeout server 30s
>
> One bit of information I came across when I first started setting
> haproxy up for Solr is that servlet containers like Jetty and Tomcat
> require the "http-pretend-keepalive" option to work properly. Are you
> using this option?
>
> Thanks,
> Shawn
>
--
------------------------------------------------------------
Nathan Neulinger nneul@neulinger.org
Neulinger Consulting (573) 612-1412
Re: /solr/admin/ping causing exceptions in log?
Posted by Shawn Heisey <so...@elyograg.org>.
On 7/27/2014 7:23 PM, Nathan Neulinger wrote:
> Unfortunately, doesn't look like this clears the symptom.
>
> The ping is responding almost instantly every time. I've tried setting a
> 15 second timeout on the check, with no change in occurences of the error.
>
> Looking at a packet capture on the server side, there is a clear
> distinction between working and failing/error-triggering connections.
>
> It looks like in a "working" case, I see two packets immediately back to
> back (one with header, and next a continuation with content) with no ack
> in between, followed by ack, rst+ack, rst.
>
> In the failing request, I see the GET request, acked, then the http/1.1
> 200 Ok response from Solr, a single ack, and then an almost
> instantaneous reset sent by the client.
>
>
> I'm only seeing this on traffic to/from haproxy checks. If I do a simple:
>
> while [ true ]; do curl -s http://host:8983/solr/admin/ping; done
>
> from the same box, that flood runs with generally 10-20ms request times
> and zero errors.
I won't claim to understand what's going on here, but it might be a
matter of the haproxy options. Here are the options I'm using in the
"defaults" section of the config:
defaults
log global
mode http
option httplog
option dontlognull
option redispatch
option abortonclose
option http-server-close
option http-pretend-keepalive
retries 1
maxconn 1024
timeout connect 1s
timeout client 5s
timeout server 30s
One bit of information I came across when I first started setting
haproxy up for Solr is that servlet containers like Jetty and Tomcat
require the "http-pretend-keepalive" option to work properly. Are you
using this option?
Thanks,
Shawn
Re: /solr/admin/ping causing exceptions in log?
Posted by Nathan Neulinger <nn...@neulinger.org>.
Either way, looks like this is not a SOLR issue, but rather haproxy.
Thanks.
-- Nathan
On 07/27/2014 08:23 PM, Nathan Neulinger wrote:
> Unfortunately, doesn't look like this clears the symptom.
>
> The ping is responding almost instantly every time. I've tried setting a 15 second timeout on the check, with no change
> in occurences of the error.
>
> Looking at a packet capture on the server side, there is a clear distinction between working and
> failing/error-triggering connections.
>
> It looks like in a "working" case, I see two packets immediately back to back (one with header, and next a continuation
> with content) with no ack in between, followed by ack, rst+ack, rst.
>
> In the failing request, I see the GET request, acked, then the http/1.1 200 Ok response from Solr, a single ack, and
> then an almost instantaneous reset sent by the client.
>
>
> I'm only seeing this on traffic to/from haproxy checks. If I do a simple:
>
> while [ true ]; do curl -s http://host:8983/solr/admin/ping; done
>
> from the same box, that flood runs with generally 10-20ms request times and zero errors.
>
> -- Nathan
>
> On 07/27/2014 07:12 PM, Nathan Neulinger wrote:
>> Cool. That's likely exactly it, since I don't have one set, it's using the check interval, and occasionally must just be
>> too short.
>>
>> Thank you!
>>
>> -- Nathan
>>
>>
>>> I assume that this is the httpchk config to make sure that the server is
>>> operational. If so, you need to increase the "timeout check" value,
>>> because it is too small. The ping request is taking longer to run than
>>> you have allowed in the timeout. Here's part of my haproxy config:
>>
>>
>
--
------------------------------------------------------------
Nathan Neulinger nneul@neulinger.org
Neulinger Consulting (573) 612-1412
Re: /solr/admin/ping causing exceptions in log?
Posted by Nathan Neulinger <nn...@neulinger.org>.
Unfortunately, doesn't look like this clears the symptom.
The ping is responding almost instantly every time. I've tried setting a 15 second timeout on the check, with no change
in occurences of the error.
Looking at a packet capture on the server side, there is a clear distinction between working and
failing/error-triggering connections.
It looks like in a "working" case, I see two packets immediately back to back (one with header, and next a continuation
with content) with no ack in between, followed by ack, rst+ack, rst.
In the failing request, I see the GET request, acked, then the http/1.1 200 Ok response from Solr, a single ack, and
then an almost instantaneous reset sent by the client.
I'm only seeing this on traffic to/from haproxy checks. If I do a simple:
while [ true ]; do curl -s http://host:8983/solr/admin/ping; done
from the same box, that flood runs with generally 10-20ms request times and zero errors.
-- Nathan
On 07/27/2014 07:12 PM, Nathan Neulinger wrote:
> Cool. That's likely exactly it, since I don't have one set, it's using the check interval, and occasionally must just be
> too short.
>
> Thank you!
>
> -- Nathan
>
>
>> I assume that this is the httpchk config to make sure that the server is
>> operational. If so, you need to increase the "timeout check" value,
>> because it is too small. The ping request is taking longer to run than
>> you have allowed in the timeout. Here's part of my haproxy config:
>
>
--
------------------------------------------------------------
Nathan Neulinger nneul@neulinger.org
Neulinger Consulting (573) 612-1412
Re: /solr/admin/ping causing exceptions in log?
Posted by Nathan Neulinger <nn...@neulinger.org>.
Cool. That's likely exactly it, since I don't have one set, it's using the check interval, and occasionally must just be
too short.
Thank you!
-- Nathan
> I assume that this is the httpchk config to make sure that the server is
> operational. If so, you need to increase the "timeout check" value,
> because it is too small. The ping request is taking longer to run than
> you have allowed in the timeout. Here's part of my haproxy config:
--
------------------------------------------------------------
Nathan Neulinger nneul@neulinger.org
Neulinger Consulting (573) 612-1412
Re: /solr/admin/ping causing exceptions in log?
Posted by Shawn Heisey <so...@elyograg.org>.
On 7/26/2014 5:15 PM, Nathan Neulinger wrote:
> Recently deployed haproxy in front of my solr instances, and seeing a
> large number of exceptions in the logs now... Example below. I can pound
> the server with requests against /solr/admin/ping via curl, with no
> obvious issue, but the haproxy checks appear to be aggravating something.
>
> Solr 4.8.0 w/ solr cloud, 2 nodes, 3 zk, linux x86_64
>
> It seems like when the issue occurs, I get a set of the errors all in a
> burst (below), never just one.
>
> Suggestions?
>
> -- Nathan
>
> ------------------------------------------------------------
> Nathan Neulinger nneul@neulinger.org
> Neulinger Consulting (573) 612-1412
>
>
>
> 2014-07-26 23:04:36,506 ERROR qtp1532385072-4864
> [g.apache.solr.servlet.SolrDispatchFilter] -
> null:org.eclipse.jetty.io.EofException
EofException means that the client has disconnected the TCP connection
before Solr has responded to the request.
I assume that this is the httpchk config to make sure that the server is
operational. If so, you need to increase the "timeout check" value,
because it is too small. The ping request is taking longer to run than
you have allowed in the timeout. Here's part of my haproxy config:
listen idx_nc
bind 0.0.0.0:8984
option httpchk GET /solr/ncmain/admin/ping
balance leastconn
timeout check 4990
server idxa1 10.100.0.240:8981 check inter 5s fastinter 2s rise
3 fall 2 weight 100
server idxb1 10.100.0.241:8981 check inter 5s fastinter 2s rise
3 fall 2 weight 100 backup
server idxa2 10.100.0.242:8981 check inter 15s fastinter 2s
rise 2 fall 1 weight 2 backup
server idxb2 10.100.0.243:8981 check inter 15s fastinter 2s
rise 2 fall 1 weight 1 backup
If you have allowed what you think is plenty of time, then you may need
to investigate Solr's performance or the specific query that you are
using for the ping.
http://wiki.apache.org/solr/SolrPerformanceProblems
Thanks,
Shawn
Re: /solr/admin/ping causing exceptions in log?
Posted by Nathan Neulinger <nn...@neulinger.org>.
Tried changing to use /solr/admin/cores instead as a test - still see the
same issue, though much less frequent.
------------------------------------------------------------
Nathan Neulinger nneul@neulinger.org
Neulinger Consulting (573) 612-1412
On Sat, Jul 26, 2014 at 6:15 PM, Nathan Neulinger <nn...@neulinger.org>
wrote:
> Recently deployed haproxy in front of my solr instances, and seeing a
> large number of exceptions in the logs now... Example below. I can pound
> the server with requests against /solr/admin/ping via curl, with no obvious
> issue, but the haproxy checks appear to be aggravating something.
>
> Solr 4.8.0 w/ solr cloud, 2 nodes, 3 zk, linux x86_64
>
> It seems like when the issue occurs, I get a set of the errors all in a
> burst (below), never just one.
>
> Suggestions?
>
> -- Nathan
>
> ------------------------------------------------------------
> Nathan Neulinger nneul@neulinger.org
> Neulinger Consulting (573) 612-1412
>
>
>
> 2014-07-26 23:04:36,506 ERROR qtp1532385072-4864 [g.apache.solr.servlet.SolrDispatchFilter]
> - null:org.eclipse.jetty.io.EofException
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(
> HttpGenerator.java:914)
> at org.eclipse.jetty.http.AbstractGenerator.flush(
> AbstractGenerator.java:443)
> at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100)
> at org.eclipse.jetty.server.AbstractHttpConnection$Output.
> flush(AbstractHttpConnection.java:1094)
> at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
> at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
> at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
> at org.apache.solr.util.FastWriter.flush(FastWriter.java:137)
> at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(
> SolrDispatchFilter.java:763)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:431)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:339)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:207)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1419)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(
> ServletHandler.java:455)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:137)
> at org.eclipse.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:557)
> at org.eclipse.jetty.server.session.SessionHandler.
> doHandle(SessionHandler.java:231)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doHandle(ContextHandler.java:1075)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(
> ServletHandler.java:384)
> at org.eclipse.jetty.server.session.SessionHandler.
> doScope(SessionHandler.java:193)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doScope(ContextHandler.java:1009)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:135)
> at org.eclipse.jetty.server.handler.ContextHandlerCollection.
> handle(ContextHandlerCollection.java:255)
> at org.eclipse.jetty.server.handler.HandlerCollection.
> handle(HandlerCollection.java:154)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(
> HandlerWrapper.java:116)
> at org.eclipse.jetty.server.Server.handle(Server.java:368)
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(
> AbstractHttpConnection.java:489)
> at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(
> BlockingHttpConnection.java:53)
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(
> AbstractHttpConnection.java:942)
> at org.eclipse.jetty.server.AbstractHttpConnection$
> RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
> at org.eclipse.jetty.http.HttpParser.parseNext(
> HttpParser.java:640)
> at org.eclipse.jetty.http.HttpParser.parseAvailable(
> HttpParser.java:235)
> at org.eclipse.jetty.server.BlockingHttpConnection.handle(
> BlockingHttpConnection.java:72)
> at org.eclipse.jetty.server.bio.SocketConnector$
> ConnectorEndPoint.run(SocketConnector.java:264)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(
> QueuedThreadPool.java:608)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(
> QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.net.SocketException: Connection reset
> at java.net.SocketOutputStream.socketWrite(
> SocketOutputStream.java:118)
> at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
> at org.eclipse.jetty.io.ByteArrayBuffer.writeTo(
> ByteArrayBuffer.java:375)
> at org.eclipse.jetty.io.bio.StreamEndPoint.flush(
> StreamEndPoint.java:164)
> at org.eclipse.jetty.io.bio.StreamEndPoint.flush(
> StreamEndPoint.java:194)
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(
> HttpGenerator.java:838)
> ... 36 more
>
> 2014-07-26 23:04:36,513 ERROR qtp1532385072-4864 [g.apache.solr.servlet.SolrDispatchFilter]
> - null:org.eclipse.jetty.io.EofException
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(
> HttpGenerator.java:914)
> at org.eclipse.jetty.http.AbstractGenerator.flush(
> AbstractGenerator.java:443)
> at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100)
> at org.eclipse.jetty.server.AbstractHttpConnection$Output.
> flush(AbstractHttpConnection.java:1094)
> at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
> at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
> at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
> at org.apache.solr.util.FastWriter.flush(FastWriter.java:137)
> at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(
> SolrDispatchFilter.java:763)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:431)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:339)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:207)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1419)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(
> ServletHandler.java:455)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:137)
> at org.eclipse.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:557)
> at org.eclipse.jetty.server.session.SessionHandler.
> doHandle(SessionHandler.java:231)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doHandle(ContextHandler.java:1075)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(
> ServletHandler.java:384)
> at org.eclipse.jetty.server.session.SessionHandler.
> doScope(SessionHandler.java:193)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doScope(ContextHandler.java:1009)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:135)
> at org.eclipse.jetty.server.handler.ContextHandlerCollection.
> handle(ContextHandlerCollection.java:255)
> at org.eclipse.jetty.server.handler.HandlerCollection.
> handle(HandlerCollection.java:154)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(
> HandlerWrapper.java:116)
> at org.eclipse.jetty.server.Server.handle(Server.java:368)
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(
> AbstractHttpConnection.java:489)
> at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(
> BlockingHttpConnection.java:53)
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(
> AbstractHttpConnection.java:942)
> at org.eclipse.jetty.server.AbstractHttpConnection$
> RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
> at org.eclipse.jetty.http.HttpParser.parseNext(
> HttpParser.java:640)
> at org.eclipse.jetty.http.HttpParser.parseAvailable(
> HttpParser.java:235)
> at org.eclipse.jetty.server.BlockingHttpConnection.handle(
> BlockingHttpConnection.java:72)
> at org.eclipse.jetty.server.bio.SocketConnector$
> ConnectorEndPoint.run(SocketConnector.java:264)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(
> QueuedThreadPool.java:608)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(
> QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.net.SocketException: Connection reset
> at java.net.SocketOutputStream.socketWrite(
> SocketOutputStream.java:118)
> at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
> at org.eclipse.jetty.io.ByteArrayBuffer.writeTo(
> ByteArrayBuffer.java:375)
> at org.eclipse.jetty.io.bio.StreamEndPoint.flush(
> StreamEndPoint.java:164)
> at org.eclipse.jetty.io.bio.StreamEndPoint.flush(
> StreamEndPoint.java:194)
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(
> HttpGenerator.java:838)
> ... 36 more
>
> 2014-07-26 23:04:36,514 WARN qtp1532385072-4864 [org.eclipse.jetty.server.Response
> ] - Committed before 500 {msg=Connection
> reset,trace=org.eclipse.jetty.io.EofException
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(
> HttpGenerator.java:914)
> at org.eclipse.jetty.http.AbstractGenerator.flush(
> AbstractGenerator.java:443)
> at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100)
> at org.eclipse.jetty.server.AbstractHttpConnection$Output.
> flush(AbstractHttpConnection.java:1094)
> at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297)
> at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
> at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
> at org.apache.solr.util.FastWriter.flush(FastWriter.java:137)
> at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(
> SolrDispatchFilter.java:763)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:431)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:339)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:207)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1419)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(
> ServletHandler.java:455)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:137)
> at org.eclipse.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:557)
> at org.eclipse.jetty.server.session.SessionHandler.
> doHandle(SessionHandler.java:231)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doHandle(ContextHandler.java:1075)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(
> ServletHandler.java:384)
> at org.eclipse.jetty.server.session.SessionHandler.
> doScope(SessionHandler.java:193)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doScope(ContextHandler.java:1009)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:135)
> at org.eclipse.jetty.server.handler.ContextHandlerCollection.
> handle(ContextHandlerCollection.java:255)
> at org.eclipse.jetty.server.handler.HandlerCollection.
> handle(HandlerCollection.java:154)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(
> HandlerWrapper.java:116)
> at org.eclipse.jetty.server.Server.handle(Server.java:368)
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(
> AbstractHttpConnection.java:489)
> at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(
> BlockingHttpConnection.java:53)
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(
> AbstractHttpConnection.java:942)
> at org.eclipse.jetty.server.AbstractHttpConnection$
> RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
> at org.eclipse.jetty.http.HttpParser.parseNext(
> HttpParser.java:640)
> at org.eclipse.jetty.http.HttpParser.parseAvailable(
> HttpParser.java:235)
> at org.eclipse.jetty.server.BlockingHttpConnection.handle(
> BlockingHttpConnection.java:72)
> at org.eclipse.jetty.server.bio.SocketConnector$
> ConnectorEndPoint.run(SocketConnector.java:264)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(
> QueuedThreadPool.java:608)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(
> QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.net.SocketException: Connection reset
> at java.net.SocketOutputStream.socketWrite(
> SocketOutputStream.java:118)
> at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
> at org.eclipse.jetty.io.ByteArrayBuffer.writeTo(
> ByteArrayBuffer.java:375)
> at org.eclipse.jetty.io.bio.StreamEndPoint.flush(
> StreamEndPoint.java:164)
> at org.eclipse.jetty.io.bio.StreamEndPoint.flush(
> StreamEndPoint.java:194)
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(
> HttpGenerator.java:838)
> ... 36 more
> ,code=500}
> 2014-07-26 23:04:36,517 ERROR qtp1532385072-4864 [g.apache.solr.servlet.SolrDispatchFilter]
> - null:java.lang.IllegalStateException: Committed
> at org.eclipse.jetty.server.Response.resetBuffer(Response.
> java:1144)
> at org.eclipse.jetty.server.Response.sendError(Response.java:314)
> at org.apache.solr.servlet.SolrDispatchFilter.sendError(
> SolrDispatchFilter.java:818)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:439)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:339)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:207)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1419)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(
> ServletHandler.java:455)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:137)
> at org.eclipse.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:557)
> at org.eclipse.jetty.server.session.SessionHandler.
> doHandle(SessionHandler.java:231)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doHandle(ContextHandler.java:1075)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(
> ServletHandler.java:384)
> at org.eclipse.jetty.server.session.SessionHandler.
> doScope(SessionHandler.java:193)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doScope(ContextHandler.java:1009)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:135)
> at org.eclipse.jetty.server.handler.ContextHandlerCollection.
> handle(ContextHandlerCollection.java:255)
> at org.eclipse.jetty.server.handler.HandlerCollection.
> handle(HandlerCollection.java:154)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(
> HandlerWrapper.java:116)
> at org.eclipse.jetty.server.Server.handle(Server.java:368)
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(
> AbstractHttpConnection.java:489)
> at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(
> BlockingHttpConnection.java:53)
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(
> AbstractHttpConnection.java:942)
> at org.eclipse.jetty.server.AbstractHttpConnection$
> RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
> at org.eclipse.jetty.http.HttpParser.parseNext(
> HttpParser.java:640)
> at org.eclipse.jetty.http.HttpParser.parseAvailable(
> HttpParser.java:235)
> at org.eclipse.jetty.server.BlockingHttpConnection.handle(
> BlockingHttpConnection.java:72)
> at org.eclipse.jetty.server.bio.SocketConnector$
> ConnectorEndPoint.run(SocketConnector.java:264)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(
> QueuedThreadPool.java:608)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(
> QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> 2014-07-26 23:04:36,518 WARN qtp1532385072-4864 [org.eclipse.jetty.server.Response
> ] - Committed before 500 {msg=Committed,trace=java.lang.IllegalStateException:
> Committed
> at org.eclipse.jetty.server.Response.resetBuffer(Response.
> java:1144)
> at org.eclipse.jetty.server.Response.sendError(Response.java:314)
> at org.apache.solr.servlet.SolrDispatchFilter.sendError(
> SolrDispatchFilter.java:818)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:439)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:339)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:207)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1419)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(
> ServletHandler.java:455)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:137)
> at org.eclipse.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:557)
> at org.eclipse.jetty.server.session.SessionHandler.
> doHandle(SessionHandler.java:231)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doHandle(ContextHandler.java:1075)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(
> ServletHandler.java:384)
> at org.eclipse.jetty.server.session.SessionHandler.
> doScope(SessionHandler.java:193)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doScope(ContextHandler.java:1009)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:135)
> at org.eclipse.jetty.server.handler.ContextHandlerCollection.
> handle(ContextHandlerCollection.java:255)
> at org.eclipse.jetty.server.handler.HandlerCollection.
> handle(HandlerCollection.java:154)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(
> HandlerWrapper.java:116)
> at org.eclipse.jetty.server.Server.handle(Server.java:368)
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(
> AbstractHttpConnection.java:489)
> at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(
> BlockingHttpConnection.java:53)
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(
> AbstractHttpConnection.java:942)
> at org.eclipse.jetty.server.AbstractHttpConnection$
> RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
> at org.eclipse.jetty.http.HttpParser.parseNext(
> HttpParser.java:640)
> at org.eclipse.jetty.http.HttpParser.parseAvailable(
> HttpParser.java:235)
> at org.eclipse.jetty.server.BlockingHttpConnection.handle(
> BlockingHttpConnection.java:72)
> at org.eclipse.jetty.server.bio.SocketConnector$
> ConnectorEndPoint.run(SocketConnector.java:264)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(
> QueuedThreadPool.java:608)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(
> QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
> ,code=500}
> 2014-07-26 23:04:36,518 WARN qtp1532385072-4864
> [org.eclipse.jetty.servlet.ServletHandler] - /solr/admin/ping
> java.lang.IllegalStateException: Committed
> at org.eclipse.jetty.server.Response.resetBuffer(Response.
> java:1144)
> at org.eclipse.jetty.server.Response.sendError(Response.java:314)
> at org.apache.solr.servlet.SolrDispatchFilter.sendError(
> SolrDispatchFilter.java:818)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:439)
> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:207)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1419)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(
> ServletHandler.java:455)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:137)
> at org.eclipse.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:557)
> at org.eclipse.jetty.server.session.SessionHandler.
> doHandle(SessionHandler.java:231)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doHandle(ContextHandler.java:1075)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(
> ServletHandler.java:384)
> at org.eclipse.jetty.server.session.SessionHandler.
> doScope(SessionHandler.java:193)
> at org.eclipse.jetty.server.handler.ContextHandler.
> doScope(ContextHandler.java:1009)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:135)
> at org.eclipse.jetty.server.handler.ContextHandlerCollection.
> handle(ContextHandlerCollection.java:255)
> at org.eclipse.jetty.server.handler.HandlerCollection.
> handle(HandlerCollection.java:154)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(
> HandlerWrapper.java:116)
> at org.eclipse.jetty.server.Server.handle(Server.java:368)
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(
> AbstractHttpConnection.java:489)
> at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(
> BlockingHttpConnection.java:53)
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(
> AbstractHttpConnection.java:942)
> at org.eclipse.jetty.server.AbstractHttpConnection$
> RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
> at org.eclipse.jetty.http.HttpParser.parseNext(
> HttpParser.java:640)
> at org.eclipse.jetty.http.HttpParser.parseAvailable(
> HttpParser.java:235)
> at org.eclipse.jetty.server.BlockingHttpConnection.handle(
> BlockingHttpConnection.java:72)
> at org.eclipse.jetty.server.bio.SocketConnector$
> ConnectorEndPoint.run(SocketConnector.java:264)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(
> QueuedThreadPool.java:608)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(
> QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>