You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2015/11/13 03:33:38 UTC

[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_66) - Build # 14887 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/14887/
Java: 32bit/jdk1.8.0_66 -client -XX:+UseParallelGC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestCloudPivotFacet

Error Message:
12 threads leaked from SUITE scope at org.apache.solr.cloud.TestCloudPivotFacet:     1) Thread[id=10145, name=qtp22576580-10145, state=BLOCKED, group=TGRP-TestCloudPivotFacet]         at org.apache.lucene.uninverting.FieldCacheImpl$Cache.get(FieldCacheImpl.java:170)         at org.apache.lucene.uninverting.FieldCacheImpl.getDocTermOrds(FieldCacheImpl.java:923)         at org.apache.lucene.uninverting.UninvertingReader.getSortedSetDocValues(UninvertingReader.java:273)         at org.apache.lucene.index.FilterLeafReader.getSortedSetDocValues(FilterLeafReader.java:459)         at org.apache.lucene.index.SlowCompositeReaderWrapper.getSortedSetDocValues(SlowCompositeReaderWrapper.java:185)         at org.apache.solr.request.DocValuesStats.getCounts(DocValuesStats.java:87)         at org.apache.solr.handler.component.StatsField.computeLocalStatsValues(StatsField.java:424)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:328)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=10230, name=qtp12050735-10230, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]         at java.lang.Object.clone(Native Method)         at org.apache.lucene.store.DataInput.clone(DataInput.java:253)         at org.apache.lucene.store.IndexInput.clone(IndexInput.java:93)         at org.apache.lucene.store.MockIndexInputWrapper.clone(MockIndexInputWrapper.java:78)         at org.apache.lucene.store.MockIndexInputWrapper.clone(MockIndexInputWrapper.java:30)         at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.initIndexInput(SegmentTermsEnum.java:115)         at org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:148)         at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)         at org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:153)         at org.apache.lucene.search.TermQuery$TermWeight.termNotInReader(TermQuery.java:132)         at org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:117)         at org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:101)         at org.apache.lucene.search.Weight.bulkScorer(Weight.java:135)         at org.apache.lucene.search.BooleanWeight.booleanScorer(BooleanWeight.java:198)         at org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:239)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)         at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)         at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)         at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)         at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:1316)         at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:375)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=10020, name=searcherExecutor-4499-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=9973, name=qtp12050735-9973, state=BLOCKED, group=TGRP-TestCloudPivotFacet]         at org.apache.lucene.uninverting.FieldCacheImpl$Cache.get(FieldCacheImpl.java:170)         at org.apache.lucene.uninverting.FieldCacheImpl.getDocTermOrds(FieldCacheImpl.java:923)         at org.apache.lucene.uninverting.UninvertingReader.getSortedSetDocValues(UninvertingReader.java:273)         at org.apache.lucene.index.FilterLeafReader.getSortedSetDocValues(FilterLeafReader.java:459)         at org.apache.lucene.index.SlowCompositeReaderWrapper.getSortedSetDocValues(SlowCompositeReaderWrapper.java:185)         at org.apache.solr.request.DocValuesStats.getCounts(DocValuesStats.java:87)         at org.apache.solr.handler.component.StatsField.computeLocalStatsValues(StatsField.java:424)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:328)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=10144, name=qtp23995526-10144, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]         at org.apache.lucene.search.ConstantScoreScorer.nextDoc(ConstantScoreScorer.java:78)         at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:216)         at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:169)         at org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)         at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)         at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)         at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)         at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:930)         at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:371)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=9944, name=qtp11658664-9944, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]         at org.apache.solr.search.DocSetCollector.collect(DocSetCollector.java:69)         at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:218)         at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:169)         at org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)         at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)         at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)         at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)         at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:930)         at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:371)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=10235, name=qtp11658664-10235, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]         at java.lang.Class.getSuperclass(Native Method)         at java.util.RegularEnumSet.contains(RegularEnumSet.java:144)         at org.apache.solr.handler.component.StatsField.includeInResponse(StatsField.java:569)         at org.apache.solr.handler.component.AbstractStatsValues.getStatsValues(StatsValuesFactory.java:347)         at org.apache.solr.handler.component.StatsComponent.convertToResponse(StatsComponent.java:151)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:330)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=9991, name=searcherExecutor-4488-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=9963, name=searcherExecutor-4477-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=10051, name=searcherExecutor-4510-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=9946, name=qtp11658664-9946, state=BLOCKED, group=TGRP-TestCloudPivotFacet]         at org.apache.lucene.uninverting.FieldCacheImpl$Cache.get(FieldCacheImpl.java:170)         at org.apache.lucene.uninverting.FieldCacheImpl.getDocTermOrds(FieldCacheImpl.java:923)         at org.apache.lucene.uninverting.UninvertingReader.getSortedSetDocValues(UninvertingReader.java:273)         at org.apache.lucene.index.FilterLeafReader.getSortedSetDocValues(FilterLeafReader.java:459)         at org.apache.lucene.index.SlowCompositeReaderWrapper.getSortedSetDocValues(SlowCompositeReaderWrapper.java:185)         at org.apache.solr.request.DocValuesStats.getCounts(DocValuesStats.java:87)         at org.apache.solr.handler.component.StatsField.computeLocalStatsValues(StatsField.java:424)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:328)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=9972, name=qtp12050735-9972, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]         at org.apache.solr.search.DocSetCollector.collect(DocSetCollector.java:69)         at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:218)         at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:169)         at org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)         at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)         at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)         at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)         at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:930)         at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:371)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 12 threads leaked from SUITE scope at org.apache.solr.cloud.TestCloudPivotFacet: 
   1) Thread[id=10145, name=qtp22576580-10145, state=BLOCKED, group=TGRP-TestCloudPivotFacet]
        at org.apache.lucene.uninverting.FieldCacheImpl$Cache.get(FieldCacheImpl.java:170)
        at org.apache.lucene.uninverting.FieldCacheImpl.getDocTermOrds(FieldCacheImpl.java:923)
        at org.apache.lucene.uninverting.UninvertingReader.getSortedSetDocValues(UninvertingReader.java:273)
        at org.apache.lucene.index.FilterLeafReader.getSortedSetDocValues(FilterLeafReader.java:459)
        at org.apache.lucene.index.SlowCompositeReaderWrapper.getSortedSetDocValues(SlowCompositeReaderWrapper.java:185)
        at org.apache.solr.request.DocValuesStats.getCounts(DocValuesStats.java:87)
        at org.apache.solr.handler.component.StatsField.computeLocalStatsValues(StatsField.java:424)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:328)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
        at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
        at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
        at org.eclipse.jetty.server.Server.handle(Server.java:499)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
        at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=10230, name=qtp12050735-10230, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
        at java.lang.Object.clone(Native Method)
        at org.apache.lucene.store.DataInput.clone(DataInput.java:253)
        at org.apache.lucene.store.IndexInput.clone(IndexInput.java:93)
        at org.apache.lucene.store.MockIndexInputWrapper.clone(MockIndexInputWrapper.java:78)
        at org.apache.lucene.store.MockIndexInputWrapper.clone(MockIndexInputWrapper.java:30)
        at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.initIndexInput(SegmentTermsEnum.java:115)
        at org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:148)
        at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)
        at org.apache.lucene.index.LeafReader.docFreq(LeafReader.java:153)
        at org.apache.lucene.search.TermQuery$TermWeight.termNotInReader(TermQuery.java:132)
        at org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:117)
        at org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:101)
        at org.apache.lucene.search.Weight.bulkScorer(Weight.java:135)
        at org.apache.lucene.search.BooleanWeight.booleanScorer(BooleanWeight.java:198)
        at org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:239)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)
        at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)
        at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)
        at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)
        at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:1316)
        at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:375)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
        at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
        at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
        at org.eclipse.jetty.server.Server.handle(Server.java:499)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
        at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=10020, name=searcherExecutor-4499-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=9973, name=qtp12050735-9973, state=BLOCKED, group=TGRP-TestCloudPivotFacet]
        at org.apache.lucene.uninverting.FieldCacheImpl$Cache.get(FieldCacheImpl.java:170)
        at org.apache.lucene.uninverting.FieldCacheImpl.getDocTermOrds(FieldCacheImpl.java:923)
        at org.apache.lucene.uninverting.UninvertingReader.getSortedSetDocValues(UninvertingReader.java:273)
        at org.apache.lucene.index.FilterLeafReader.getSortedSetDocValues(FilterLeafReader.java:459)
        at org.apache.lucene.index.SlowCompositeReaderWrapper.getSortedSetDocValues(SlowCompositeReaderWrapper.java:185)
        at org.apache.solr.request.DocValuesStats.getCounts(DocValuesStats.java:87)
        at org.apache.solr.handler.component.StatsField.computeLocalStatsValues(StatsField.java:424)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:328)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
        at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
        at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
        at org.eclipse.jetty.server.Server.handle(Server.java:499)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
        at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=10144, name=qtp23995526-10144, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
        at org.apache.lucene.search.ConstantScoreScorer.nextDoc(ConstantScoreScorer.java:78)
        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:216)
        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:169)
        at org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)
        at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)
        at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)
        at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)
        at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:930)
        at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:371)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
        at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
        at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
        at org.eclipse.jetty.server.Server.handle(Server.java:499)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
        at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=9944, name=qtp11658664-9944, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
        at org.apache.solr.search.DocSetCollector.collect(DocSetCollector.java:69)
        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:218)
        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:169)
        at org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)
        at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)
        at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)
        at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)
        at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:930)
        at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:371)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
        at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
        at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
        at org.eclipse.jetty.server.Server.handle(Server.java:499)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
        at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=10235, name=qtp11658664-10235, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
        at java.lang.Class.getSuperclass(Native Method)
        at java.util.RegularEnumSet.contains(RegularEnumSet.java:144)
        at org.apache.solr.handler.component.StatsField.includeInResponse(StatsField.java:569)
        at org.apache.solr.handler.component.AbstractStatsValues.getStatsValues(StatsValuesFactory.java:347)
        at org.apache.solr.handler.component.StatsComponent.convertToResponse(StatsComponent.java:151)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:330)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
        at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
        at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
        at org.eclipse.jetty.server.Server.handle(Server.java:499)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
        at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=9991, name=searcherExecutor-4488-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=9963, name=searcherExecutor-4477-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=10051, name=searcherExecutor-4510-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=9946, name=qtp11658664-9946, state=BLOCKED, group=TGRP-TestCloudPivotFacet]
        at org.apache.lucene.uninverting.FieldCacheImpl$Cache.get(FieldCacheImpl.java:170)
        at org.apache.lucene.uninverting.FieldCacheImpl.getDocTermOrds(FieldCacheImpl.java:923)
        at org.apache.lucene.uninverting.UninvertingReader.getSortedSetDocValues(UninvertingReader.java:273)
        at org.apache.lucene.index.FilterLeafReader.getSortedSetDocValues(FilterLeafReader.java:459)
        at org.apache.lucene.index.SlowCompositeReaderWrapper.getSortedSetDocValues(SlowCompositeReaderWrapper.java:185)
        at org.apache.solr.request.DocValuesStats.getCounts(DocValuesStats.java:87)
        at org.apache.solr.handler.component.StatsField.computeLocalStatsValues(StatsField.java:424)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:328)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
        at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
        at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
        at org.eclipse.jetty.server.Server.handle(Server.java:499)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
        at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=9972, name=qtp12050735-9972, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
        at org.apache.solr.search.DocSetCollector.collect(DocSetCollector.java:69)
        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:218)
        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:169)
        at org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:670)
        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)
        at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)
        at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)
        at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)
        at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:930)
        at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:371)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
        at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
        at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
        at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
        at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
        at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
        at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
        at org.eclipse.jetty.server.Server.handle(Server.java:499)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
        at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([96AD26151EF21E82]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestCloudPivotFacet

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=10145, name=qtp22576580-10145, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]         at org.apache.lucene.codecs.lucene50.Lucene50PostingsReader$BlockDocsEnum.<init>(Lucene50PostingsReader.java:285)         at org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.postings(Lucene50PostingsReader.java:208)         at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.postings(SegmentTermsEnum.java:1002)         at org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:105)         at org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:280)         at org.apache.lucene.search.Weight.bulkScorer(Weight.java:135)         at org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:262)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)         at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)         at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)         at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)         at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:1316)         at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:375)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=10230, name=qtp12050735-10230, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]         at java.util.Arrays.copyOfRange(Arrays.java:3664)         at java.lang.String.<init>(String.java:207)         at java.lang.StringBuilder.toString(StringBuilder.java:407)         at org.apache.lucene.store.MockIndexInputWrapper.<init>(MockIndexInputWrapper.java:39)         at org.apache.lucene.store.MockIndexInputWrapper.clone(MockIndexInputWrapper.java:79)         at org.apache.lucene.store.MockIndexInputWrapper.clone(MockIndexInputWrapper.java:30)         at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.initIndexInput(SegmentTermsEnum.java:115)         at org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:148)         at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)         at org.apache.lucene.index.TermContext.build(TermContext.java:94)         at org.apache.lucene.search.TermQuery.createWeight(TermQuery.java:191)         at org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:753)         at org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:56)         at org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:220)         at org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:753)         at org.apache.lucene.search.IndexSearcher.createNormalizedWeight(IndexSearcher.java:736)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)         at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)         at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)         at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)         at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:1316)         at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:375)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=10020, name=searcherExecutor-4499-thread-1, state=WAITING, group=TGRP-TestCloudPivotFacet]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=9944, name=qtp11658664-9944, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]         at org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:55)         at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:77)         at org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)         at org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:123)         at org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:101)         at org.apache.lucene.search.Weight.bulkScorer(Weight.java:135)         at org.apache.lucene.search.BooleanWeight.booleanScorer(BooleanWeight.java:198)         at org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:239)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:474)         at org.apache.solr.search.DocSetUtil.createDocSetGeneric(DocSetUtil.java:104)         at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:93)         at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)         at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:1316)         at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:375)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=9973, name=qtp12050735-9973, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]         at java.lang.Class.getComponentType(Native Method)         at java.util.Arrays.copyOf(Arrays.java:3212)         at java.util.ArrayList.toArray(ArrayList.java:407)         at org.apache.lucene.search.BooleanQuery$Builder.build(BooleanQuery.java:144)         at org.apache.solr.search.DocSetUtil.createDocSet(DocSetUtil.java:80)         at org.apache.solr.search.SolrIndexSearcher.getDocSetNC(SolrIndexSearcher.java:1274)         at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:1316)         at org.apache.solr.handler.component.PivotFacetProcessor.getSubset(PivotFacetProcessor.java:375)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:305)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)         at org.eclipse.jetty.server.Server.handle(Server.java:499)         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)         at org.eclipse.jetty.server.H

[...truncated too long message...]

sor.java:322)
   [junit4]    >         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
   [junit4]    >         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
   [junit4]    >         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
   [junit4]    >         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
   [junit4]    >         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
   [junit4]    >         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
   [junit4]    >         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
   [junit4]    >         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
   [junit4]    >         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
   [junit4]    >         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
   [junit4]    >         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
   [junit4]    >         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    >         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
   [junit4]    >         at org.eclipse.jetty.server.Server.handle(Server.java:499)
   [junit4]    >         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
   [junit4]    >         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
   [junit4]    >         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   12) Thread[id=9972, name=qtp12050735-9972, state=RUNNABLE, group=TGRP-TestCloudPivotFacet]
   [junit4]    >         at java.lang.Class.getSuperclass(Native Method)
   [junit4]    >         at java.util.RegularEnumSet.contains(RegularEnumSet.java:144)
   [junit4]    >         at org.apache.solr.handler.component.StatsField.calculateStats(StatsField.java:564)
   [junit4]    >         at org.apache.solr.handler.component.AbstractStatsValues.<init>(StatsValuesFactory.java:162)
   [junit4]    >         at org.apache.solr.handler.component.NumericStatsValues.<init>(StatsValuesFactory.java:457)
   [junit4]    >         at org.apache.solr.handler.component.StatsValuesFactory.createStatsValues(StatsValuesFactory.java:73)
   [junit4]    >         at org.apache.solr.handler.component.StatsField.computeLocalValueSourceStats(StatsField.java:436)
   [junit4]    >         at org.apache.solr.handler.component.StatsField.computeLocalStatsValues(StatsField.java:428)
   [junit4]    >         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:328)
   [junit4]    >         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
   [junit4]    >         at org.apache.solr.handler.component.PivotFacetProcessor.doPivots(PivotFacetProcessor.java:322)
   [junit4]    >         at org.apache.solr.handler.component.PivotFacetProcessor.processSingle(PivotFacetProcessor.java:228)
   [junit4]    >         at org.apache.solr.handler.component.PivotFacetProcessor.process(PivotFacetProcessor.java:170)
   [junit4]    >         at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:262)
   [junit4]    >         at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:276)
   [junit4]    >         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:151)
   [junit4]    >         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2079)
   [junit4]    >         at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:667)
   [junit4]    >         at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:179)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
   [junit4]    >         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
   [junit4]    >         at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
   [junit4]    >         at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:300)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    >         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
   [junit4]    >         at org.eclipse.jetty.server.Server.handle(Server.java:499)
   [junit4]    >         at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
   [junit4]    >         at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
   [junit4]    >         at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([96AD26151EF21E82]:0)
   [junit4] Completed [534/557] on J1 in 605.95s, 1 test, 3 errors <<< FAILURES!

[...truncated 70 lines...]
   [junit4] JVM J1: stdout was not empty, see: /home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/temp/junit4-J1-20151113_015941_225.sysout
   [junit4] >>> JVM J1: stdout (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /home/jenkins/workspace/Lucene-Solr-trunk-Linux/heapdumps/java_pid28906.hprof ...
   [junit4] Heap dump file created [491517975 bytes in 5.515 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 11 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:775: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:719: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:516: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1432: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:987: There were test failures: 557 suites (8 ignored), 2213 tests, 2 suite-level errors, 1 error, 118 ignored (69 assumptions) [seed: 96AD26151EF21E82]

Total time: 55 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any