You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@qpid.apache.org by Adel Boutros <ad...@live.com> on 2016/08/09 13:24:08 UTC
[Qpid Java Broker 6.0.4] Hanging while performing REST API calls
for management
Hello,
I am updating the broker we are using from 6.0.1 to 6.0.4 and noticed that our tests sometimes hang between 1 and 3 minutes. While they hang, I cannot access the Web Management console.
Then, when the hanging finishes, the test passes and I can access again the Web console.
The hanging is always happening when I try to create queue bindings via REST API. The hanging happens every run but on which query it hangs is random because the tests perform many REST API calls such as deleting/creating queues, deleting/creating bindings, etc. The tests are using JMS as a client API.
I activated the DEBUG level on the ROOT logger of the Broker side and noticed that when my test is hanging, the broker doesn't receive the REST request and generates the below errors multiple times until the hanging is gone. When the hanging is over, I have 2-3 other exceptions about EOF.
Exception when hanging starts:
DEBUG [Selector-Port-AMQP] (o.a.q.s.t.SelectorThread) - Failed to register with selector for connection [NonBlockingConnection /10.27.40.20:54034]. Connection is probably being closed by peer.
java.nio.channels.ClosedChannelException: null
at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:194) ~[na:1.7.0_79]
at org.apache.qpid.server.transport.SelectorThread$SelectionTask.processUnscheduledConnections(SelectorThread.java:138) [qpid-broker-core-6.0.4.jar:6.0.4]
at org.apache.qpid.server.transport.SelectorThread$SelectionTask.performSelect(SelectorThread.java:319) [qpid-broker-core-6.0.4.jar:6.0.4]
at org.apache.qpid.server.transport.SelectorThread$SelectionTask.run(SelectorThread.java:87) [qpid-broker-core-6.0.4.jar:6.0.4]
at org.apache.qpid.server.transport.SelectorThread.run(SelectorThread.java:462) [qpid-broker-core-6.0.4.jar:6.0.4]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_79]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_79]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
Regards,
Adel
Exceptions when hanging ends:
2016-08-09 15:03:51,429 DEBUG [HttpManagement-HTTP-147 - /api/latest/exchange/default/default/amq.topic] (o.a.q.s.m.p.f.ExceptionHandlingFilter) - Exception in servlet '/api/latest/exchange/default/default/amq.topic':
org.eclipse.jetty.io.EofException: timeout
at org.eclipse.jetty.http.AbstractGenerator.blockForOutput(AbstractGenerator.java:527) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:170) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1848) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeRaw(UTF8JsonGenerator.java:616) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.util.DefaultIndenter.writeIndentation(DefaultIndenter.java:91) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.util.DefaultPrettyPrinter.writeObjectEntrySeparator(DefaultPrettyPrinter.java:307) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.json.UTF8JsonGenerator._writePPFieldName(UTF8JsonGenerator.java:350) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeFieldName(UTF8JsonGenerator.java:176) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.StdKeySerializer.serialize(StdKeySerializer.java:39) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:538) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:129) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2745) ~[jackson-databind-2.5.3.jar:2.5.3]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.writeObjectToResponse(AbstractServlet.java:313) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.sendJsonResponse(AbstractServlet.java:285) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.doGetWithSubjectAndActor(RestServlet.java:478) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:86) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_79]
at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_79]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doWithSubjectAndActor(AbstractServlet.java:215) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doGet(AbstractServlet.java:84) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:575) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.service(RestServlet.java:531) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:668) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1496) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.apache.qpid.server.management.plugin.filter.ForbiddingAuthorisationFilter.doFilter(ForbiddingAuthorisationFilter.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.apache.qpid.server.management.plugin.filter.ForbiddingTraceFilter.doFilter(ForbiddingTraceFilter.java:65) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.apache.qpid.server.management.plugin.filter.LoggingFilter.doFilter(LoggingFilter.java:70) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.apache.qpid.server.management.plugin.filter.ExceptionHandlingFilter.doFilter(ExceptionHandlingFilter.java:56) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.Server.handle(Server.java:366) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
2016-08-09 15:03:51,429 DEBUG [HttpManagement-HTTP-147 - /api/latest/exchange/default/default/amq.topic] (o.e.j.s.AbstractHttpConnection) -
org.eclipse.jetty.io.EofException: timeout
at org.eclipse.jetty.http.AbstractGenerator.blockForOutput(AbstractGenerator.java:527) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:170) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1848) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeRaw(UTF8JsonGenerator.java:616) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.util.DefaultIndenter.writeIndentation(DefaultIndenter.java:91) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.util.DefaultPrettyPrinter.writeObjectEntrySeparator(DefaultPrettyPrinter.java:307) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.json.UTF8JsonGenerator._writePPFieldName(UTF8JsonGenerator.java:350) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeFieldName(UTF8JsonGenerator.java:176) ~[jackson-core-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.StdKeySerializer.serialize(StdKeySerializer.java:39) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:538) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:129) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385) ~[jackson-databind-2.5.3.jar:2.5.3]
at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2745) ~[jackson-databind-2.5.3.jar:2.5.3]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.writeObjectToResponse(AbstractServlet.java:313) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.sendJsonResponse(AbstractServlet.java:285) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.doGetWithSubjectAndActor(RestServlet.java:478) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:86) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_79]
at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_79]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doWithSubjectAndActor(AbstractServlet.java:215) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doGet(AbstractServlet.java:84) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:575) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.service(RestServlet.java:531) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:668) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1496) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.apache.qpid.server.management.plugin.filter.ForbiddingAuthorisationFilter.doFilter(ForbiddingAuthorisationFilter.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.apache.qpid.server.management.plugin.filter.ForbiddingTraceFilter.doFilter(ForbiddingTraceFilter.java:65) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.apache.qpid.server.management.plugin.filter.LoggingFilter.doFilter(LoggingFilter.java:70) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.apache.qpid.server.management.plugin.filter.ExceptionHandlingFilter.doFilter(ExceptionHandlingFilter.java:56) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.Server.handle(Server.java:366) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
2016-08-09 15:03:51,430 DEBUG [HttpManagement-HTTP-147] (o.e.j.i.nio) - EOF
org.eclipse.jetty.io.EofException: null
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:914) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.http.HttpGenerator.complete(HttpGenerator.java:798) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection.completeResponse(AbstractHttpConnection.java:677) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.Response.complete(Response.java:1228) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:611) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
Caused by: java.nio.channels.ClosedChannelException: null
at sun.nio.ch.SocketChannelImpl.ensureWriteOpen(SocketChannelImpl.java:265) ~[na:1.7.0_79]
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:474) ~[na:1.7.0_79]
at org.eclipse.jetty.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:293) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:404) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:341) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:378) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:841) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
... 14 common frames omitted
Re: [Qpid Java Broker 6.0.4] Hanging while performing REST API calls
for management
Posted by Keith W <ke...@gmail.com>.
Hi Adel,
I need to see a log from the Broker when the problems occur. I
suggest either opening a Jira and attaching a suitably scrubbed log
there, or sharing via another channel. Also I suggest take a series
of thread dumps from the JVM whilst the problem manifests (maybe one
every 10 seconds).
On the two exceptions you point out above:
1) "Connection is probably being closed by peer." - this is known and
*should* be harmless. The ordering of the shutdown of the AMQP socket
channel and removal of the selection key from the selector is not
quite correct and this exception occurs. To the best of our knowledge
there is no detrimental effect and there is no O/S level resource
leak. This is already earmarked for 6.2,
2) The Jetty timeout exceptions actually suggest a problem at the
client side. The stack suggests that Jetty's attempt to write data to
the peer is timing out. Is it an bespoke application or browser at
the other end? If it the latter you might want to turn up debug
there and check what is going on.
Kind regards, Keith Wall.
On 9 August 2016 at 14:24, Adel Boutros <ad...@live.com> wrote:
> Hello,
>
> I am updating the broker we are using from 6.0.1 to 6.0.4 and noticed that our tests sometimes hang between 1 and 3 minutes. While they hang, I cannot access the Web Management console.
> Then, when the hanging finishes, the test passes and I can access again the Web console.
>
> The hanging is always happening when I try to create queue bindings via REST API. The hanging happens every run but on which query it hangs is random because the tests perform many REST API calls such as deleting/creating queues, deleting/creating bindings, etc. The tests are using JMS as a client API.
>
> I activated the DEBUG level on the ROOT logger of the Broker side and noticed that when my test is hanging, the broker doesn't receive the REST request and generates the below errors multiple times until the hanging is gone. When the hanging is over, I have 2-3 other exceptions about EOF.
>
> Exception when hanging starts:
>
> DEBUG [Selector-Port-AMQP] (o.a.q.s.t.SelectorThread) - Failed to register with selector for connection [NonBlockingConnection /10.27.40.20:54034]. Connection is probably being closed by peer.
> java.nio.channels.ClosedChannelException: null
> at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:194) ~[na:1.7.0_79]
> at org.apache.qpid.server.transport.SelectorThread$SelectionTask.processUnscheduledConnections(SelectorThread.java:138) [qpid-broker-core-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.transport.SelectorThread$SelectionTask.performSelect(SelectorThread.java:319) [qpid-broker-core-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.transport.SelectorThread$SelectionTask.run(SelectorThread.java:87) [qpid-broker-core-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.transport.SelectorThread.run(SelectorThread.java:462) [qpid-broker-core-6.0.4.jar:6.0.4]
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_79]
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_79]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
>
>
> Regards,
> Adel
>
> Exceptions when hanging ends:
>
> 2016-08-09 15:03:51,429 DEBUG [HttpManagement-HTTP-147 - /api/latest/exchange/default/default/amq.topic] (o.a.q.s.m.p.f.ExceptionHandlingFilter) - Exception in servlet '/api/latest/exchange/default/default/amq.topic':
> org.eclipse.jetty.io.EofException: timeout
> at org.eclipse.jetty.http.AbstractGenerator.blockForOutput(AbstractGenerator.java:527) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:170) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1848) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeRaw(UTF8JsonGenerator.java:616) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.util.DefaultIndenter.writeIndentation(DefaultIndenter.java:91) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.util.DefaultPrettyPrinter.writeObjectEntrySeparator(DefaultPrettyPrinter.java:307) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator._writePPFieldName(UTF8JsonGenerator.java:350) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeFieldName(UTF8JsonGenerator.java:176) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.StdKeySerializer.serialize(StdKeySerializer.java:39) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:538) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:129) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2745) ~[jackson-databind-2.5.3.jar:2.5.3]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.writeObjectToResponse(AbstractServlet.java:313) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.sendJsonResponse(AbstractServlet.java:285) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.doGetWithSubjectAndActor(RestServlet.java:478) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:86) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_79]
> at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_79]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doWithSubjectAndActor(AbstractServlet.java:215) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doGet(AbstractServlet.java:84) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:575) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
> at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.service(RestServlet.java:531) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:668) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
> at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1496) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ForbiddingAuthorisationFilter.doFilter(ForbiddingAuthorisationFilter.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ForbiddingTraceFilter.doFilter(ForbiddingTraceFilter.java:65) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.LoggingFilter.doFilter(LoggingFilter.java:70) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ExceptionHandlingFilter.doFilter(ExceptionHandlingFilter.java:56) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.Server.handle(Server.java:366) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
> 2016-08-09 15:03:51,429 DEBUG [HttpManagement-HTTP-147 - /api/latest/exchange/default/default/amq.topic] (o.e.j.s.AbstractHttpConnection) -
> org.eclipse.jetty.io.EofException: timeout
> at org.eclipse.jetty.http.AbstractGenerator.blockForOutput(AbstractGenerator.java:527) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:170) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1848) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeRaw(UTF8JsonGenerator.java:616) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.util.DefaultIndenter.writeIndentation(DefaultIndenter.java:91) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.util.DefaultPrettyPrinter.writeObjectEntrySeparator(DefaultPrettyPrinter.java:307) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator._writePPFieldName(UTF8JsonGenerator.java:350) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeFieldName(UTF8JsonGenerator.java:176) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.StdKeySerializer.serialize(StdKeySerializer.java:39) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:538) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:129) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2745) ~[jackson-databind-2.5.3.jar:2.5.3]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.writeObjectToResponse(AbstractServlet.java:313) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.sendJsonResponse(AbstractServlet.java:285) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.doGetWithSubjectAndActor(RestServlet.java:478) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:86) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_79]
> at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_79]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doWithSubjectAndActor(AbstractServlet.java:215) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doGet(AbstractServlet.java:84) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:575) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
> at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.service(RestServlet.java:531) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:668) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
> at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1496) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ForbiddingAuthorisationFilter.doFilter(ForbiddingAuthorisationFilter.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ForbiddingTraceFilter.doFilter(ForbiddingTraceFilter.java:65) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.LoggingFilter.doFilter(LoggingFilter.java:70) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ExceptionHandlingFilter.doFilter(ExceptionHandlingFilter.java:56) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.Server.handle(Server.java:366) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
> 2016-08-09 15:03:51,430 DEBUG [HttpManagement-HTTP-147] (o.e.j.i.nio) - EOF
> org.eclipse.jetty.io.EofException: null
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:914) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpGenerator.complete(HttpGenerator.java:798) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.completeResponse(AbstractHttpConnection.java:677) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.Response.complete(Response.java:1228) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:611) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
> Caused by: java.nio.channels.ClosedChannelException: null
> at sun.nio.ch.SocketChannelImpl.ensureWriteOpen(SocketChannelImpl.java:265) ~[na:1.7.0_79]
> at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:474) ~[na:1.7.0_79]
> at org.eclipse.jetty.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:293) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:404) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:341) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:378) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:841) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> ... 14 common frames omitted
>
---------------------------------------------------------------------
To unsubscribe, e-mail: users-unsubscribe@qpid.apache.org
For additional commands, e-mail: users-help@qpid.apache.org
RE: [Qpid Java Broker 6.0.4] Hanging while performing REST API
calls for management
Posted by Adel Boutros <ad...@live.com>.
Hello guys,
Any thoughts on the below?
Regards,
Adel
> From: adelboutros@live.com
> To: users@qpid.apache.org
> Subject: [Qpid Java Broker 6.0.4] Hanging while performing REST API calls for management
> Date: Tue, 9 Aug 2016 15:24:08 +0200
>
> Hello,
>
> I am updating the broker we are using from 6.0.1 to 6.0.4 and noticed that our tests sometimes hang between 1 and 3 minutes. While they hang, I cannot access the Web Management console.
> Then, when the hanging finishes, the test passes and I can access again the Web console.
>
> The hanging is always happening when I try to create queue bindings via REST API. The hanging happens every run but on which query it hangs is random because the tests perform many REST API calls such as deleting/creating queues, deleting/creating bindings, etc. The tests are using JMS as a client API.
>
> I activated the DEBUG level on the ROOT logger of the Broker side and noticed that when my test is hanging, the broker doesn't receive the REST request and generates the below errors multiple times until the hanging is gone. When the hanging is over, I have 2-3 other exceptions about EOF.
>
> Exception when hanging starts:
>
> DEBUG [Selector-Port-AMQP] (o.a.q.s.t.SelectorThread) - Failed to register with selector for connection [NonBlockingConnection /10.27.40.20:54034]. Connection is probably being closed by peer.
> java.nio.channels.ClosedChannelException: null
> at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:194) ~[na:1.7.0_79]
> at org.apache.qpid.server.transport.SelectorThread$SelectionTask.processUnscheduledConnections(SelectorThread.java:138) [qpid-broker-core-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.transport.SelectorThread$SelectionTask.performSelect(SelectorThread.java:319) [qpid-broker-core-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.transport.SelectorThread$SelectionTask.run(SelectorThread.java:87) [qpid-broker-core-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.transport.SelectorThread.run(SelectorThread.java:462) [qpid-broker-core-6.0.4.jar:6.0.4]
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_79]
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_79]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
>
>
> Regards,
> Adel
>
> Exceptions when hanging ends:
>
> 2016-08-09 15:03:51,429 DEBUG [HttpManagement-HTTP-147 - /api/latest/exchange/default/default/amq.topic] (o.a.q.s.m.p.f.ExceptionHandlingFilter) - Exception in servlet '/api/latest/exchange/default/default/amq.topic':
> org.eclipse.jetty.io.EofException: timeout
> at org.eclipse.jetty.http.AbstractGenerator.blockForOutput(AbstractGenerator.java:527) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:170) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1848) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeRaw(UTF8JsonGenerator.java:616) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.util.DefaultIndenter.writeIndentation(DefaultIndenter.java:91) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.util.DefaultPrettyPrinter.writeObjectEntrySeparator(DefaultPrettyPrinter.java:307) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator._writePPFieldName(UTF8JsonGenerator.java:350) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeFieldName(UTF8JsonGenerator.java:176) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.StdKeySerializer.serialize(StdKeySerializer.java:39) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:538) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:129) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2745) ~[jackson-databind-2.5.3.jar:2.5.3]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.writeObjectToResponse(AbstractServlet.java:313) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.sendJsonResponse(AbstractServlet.java:285) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.doGetWithSubjectAndActor(RestServlet.java:478) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:86) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_79]
> at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_79]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doWithSubjectAndActor(AbstractServlet.java:215) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doGet(AbstractServlet.java:84) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:575) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
> at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.service(RestServlet.java:531) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:668) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
> at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1496) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ForbiddingAuthorisationFilter.doFilter(ForbiddingAuthorisationFilter.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ForbiddingTraceFilter.doFilter(ForbiddingTraceFilter.java:65) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.LoggingFilter.doFilter(LoggingFilter.java:70) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ExceptionHandlingFilter.doFilter(ExceptionHandlingFilter.java:56) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429) [jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.Server.handle(Server.java:366) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
> 2016-08-09 15:03:51,429 DEBUG [HttpManagement-HTTP-147 - /api/latest/exchange/default/default/amq.topic] (o.e.j.s.AbstractHttpConnection) -
> org.eclipse.jetty.io.EofException: timeout
> at org.eclipse.jetty.http.AbstractGenerator.blockForOutput(AbstractGenerator.java:527) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:170) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1848) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeRaw(UTF8JsonGenerator.java:616) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.util.DefaultIndenter.writeIndentation(DefaultIndenter.java:91) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.util.DefaultPrettyPrinter.writeObjectEntrySeparator(DefaultPrettyPrinter.java:307) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator._writePPFieldName(UTF8JsonGenerator.java:350) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.core.json.UTF8JsonGenerator.writeFieldName(UTF8JsonGenerator.java:176) ~[jackson-core-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.StdKeySerializer.serialize(StdKeySerializer.java:39) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:538) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:557) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:466) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:29) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serializeContents(IndexedListSerializer.java:113) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:73) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.impl.IndexedListSerializer.serialize(IndexedListSerializer.java:19) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:129) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385) ~[jackson-databind-2.5.3.jar:2.5.3]
> at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2745) ~[jackson-databind-2.5.3.jar:2.5.3]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.writeObjectToResponse(AbstractServlet.java:313) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.sendJsonResponse(AbstractServlet.java:285) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.doGetWithSubjectAndActor(RestServlet.java:478) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet$1.run(AbstractServlet.java:86) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_79]
> at javax.security.auth.Subject.doAs(Subject.java:415) ~[na:1.7.0_79]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doWithSubjectAndActor(AbstractServlet.java:215) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.apache.qpid.server.management.plugin.servlet.rest.AbstractServlet.doGet(AbstractServlet.java:84) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:575) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
> at org.apache.qpid.server.management.plugin.servlet.rest.RestServlet.service(RestServlet.java:531) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:668) ~[geronimo-servlet_3.0_spec-1.0.jar:1.0]
> at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1496) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ForbiddingAuthorisationFilter.doFilter(ForbiddingAuthorisationFilter.java:90) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ForbiddingTraceFilter.doFilter(ForbiddingTraceFilter.java:65) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.LoggingFilter.doFilter(LoggingFilter.java:70) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.apache.qpid.server.management.plugin.filter.ExceptionHandlingFilter.doFilter(ExceptionHandlingFilter.java:56) ~[qpid-broker-plugins-management-http-6.0.4.jar:6.0.4]
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1467) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429) ~[jetty-servlet-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.Server.handle(Server.java:366) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) [jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) [jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
> 2016-08-09 15:03:51,430 DEBUG [HttpManagement-HTTP-147] (o.e.j.i.nio) - EOF
> org.eclipse.jetty.io.EofException: null
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:914) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpGenerator.complete(HttpGenerator.java:798) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.completeResponse(AbstractHttpConnection.java:677) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.Response.complete(Response.java:1228) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:611) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) ~[jetty-server-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53) [jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) [jetty-util-8.1.17.v20150415.jar:8.1.17.v20150415]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_79]
> Caused by: java.nio.channels.ClosedChannelException: null
> at sun.nio.ch.SocketChannelImpl.ensureWriteOpen(SocketChannelImpl.java:265) ~[na:1.7.0_79]
> at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:474) ~[na:1.7.0_79]
> at org.eclipse.jetty.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:293) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:404) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.ChannelEndPoint.flush(ChannelEndPoint.java:341) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.flush(SelectChannelEndPoint.java:378) ~[jetty-io-8.1.17.v20150415.jar:8.1.17.v20150415]
> at org.eclipse.jetty.http.HttpGenerator.flushBuffer(HttpGenerator.java:841) ~[jetty-http-8.1.17.v20150415.jar:8.1.17.v20150415]
> ... 14 common frames omitted
>