You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/05/29 16:01:58 UTC

[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 899 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/899/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.cloud.TestDeleteCollectionOnDownNodes.deleteCollectionWithDownNodes

Error Message:
Timed out waiting for leader elections null Last available state: null

Stack Trace:
java.lang.AssertionError: Timed out waiting for leader elections
null
Last available state: null
	at __randomizedtesting.SeedInfo.seed([3543ED238D88DAA3:AB7689DBABAB962B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:265)
	at org.apache.solr.cloud.TestDeleteCollectionOnDownNodes.deleteCollectionWithDownNodes(TestDeleteCollectionOnDownNodes.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([3543ED238D88DAA3:5F91D24CD56B0A6C]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:895)
	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=letter0:lett&wt=xml
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
	... 40 more




Build Log:
[...truncated 11589 lines...]
   [junit4] Suite: org.apache.solr.util.TestMaxTokenLenTokenizer
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.util.TestMaxTokenLenTokenizer_3543ED238D88DAA3-001/init-core-data-001
   [junit4]   2> 909054 WARN  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
   [junit4]   2> 909054 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 909058 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 909058 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 909058 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 909092 WARN  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 909093 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 909119 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 909128 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 909132 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 909190 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d8b3a2e
   [junit4]   2> 909192 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d8b3a2e
   [junit4]   2> 909192 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d8b3a2e
   [junit4]   2> 909194 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 909236 WARN  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 909237 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 909254 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 909268 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 909268 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 909268 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d8b3a2e
   [junit4]   2> 909269 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 909269 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.util.TestMaxTokenLenTokenizer_3543ED238D88DAA3-001/init-core-data-001/]
   [junit4]   2> 909269 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=37.740234375, floorSegmentMB=1.333984375, forceMergeDeletesPctAllowed=5.730979218375554, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 909315 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 909315 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 909315 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=40.59375, floorSegmentMB=1.3544921875, forceMergeDeletesPctAllowed=18.848718871630712, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.21522969031176553
   [junit4]   2> 909316 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c9a80bd[collection1] main]
   [junit4]   2> 909316 WARN  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 909316 INFO  (coreLoadExecutor-4107-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 909316 INFO  (searcherExecutor-4108-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7c9a80bd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 909317 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 909318 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSingleFieldSameAnalyzers
   [junit4]   2> 909319 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 909319 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:*} 0 0
   [junit4]   2> 909319 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909319 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909321 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@6fa2b988[collection1] main]
   [junit4]   2> 909321 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909321 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6fa2b988[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):C1)))}
   [junit4]   2> 909321 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[1],commit=} 0 2
   [junit4]   2> 909322 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909322 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909322 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@5c5dffc6[collection1] main]
   [junit4]   2> 909322 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909323 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5c5dffc6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):C1) Uninverting(_1(6.7.0):C1)))}
   [junit4]   2> 909323 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[2],commit=} 0 1
   [junit4]   2> 909323 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909323 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909324 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2397d6cb[collection1] main]
   [junit4]   2> 909324 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909324 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2397d6cb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):C1) Uninverting(_1(6.7.0):C1) Uninverting(_2(6.7.0):C1)))}
   [junit4]   2> 909324 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[3],commit=} 0 1
   [junit4]   2> 909324 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909324 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909325 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@572d5dc1[collection1] main]
   [junit4]   2> 909325 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909326 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@572d5dc1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):C1) Uninverting(_1(6.7.0):C1) Uninverting(_2(6.7.0):C1) Uninverting(_3(6.7.0):C1)))}
   [junit4]   2> 909326 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[4],commit=} 0 1
   [junit4]   2> 909326 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909326 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909327 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@66dc1c08[collection1] main]
   [junit4]   2> 909328 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909328 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@66dc1c08[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):C1) Uninverting(_1(6.7.0):C1) Uninverting(_2(6.7.0):C1) Uninverting(_3(6.7.0):C1) Uninverting(_4(6.7.0):C1)))}
   [junit4]   2> 909328 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[5],commit=} 0 2
   [junit4]   2> 909328 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909328 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 909328 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909328 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 0
   [junit4]   2> 909332 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=*:*&wt=xml} hits=5 status=0 QTime=0
   [junit4]   2> 909333 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter0:l&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 909333 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter0:let&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909333 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter0:lett&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 909334 ERROR (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=letter0:lett&wt=xml
   [junit4]   2> 909334 ERROR (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=letter0:lett&wt=xml:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=letter0:lett&wt=xml
   [junit4]   2> 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
   [junit4]   2> 	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 909334 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSingleFieldSameAnalyzers
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestMaxTokenLenTokenizer -Dtests.method=testSingleFieldSameAnalyzers -Dtests.seed=3543ED238D88DAA3 -Dtests.slow=true -Dtests.locale=ga -Dtests.timezone=Asia/Irkutsk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.02s J1 | TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during query
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3543ED238D88DAA3:5F91D24CD56B0A6C]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:895)
   [junit4]    > 	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]    > 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]    > </response>
   [junit4]    > 	request was:q=letter0:lett&wt=xml
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
   [junit4]    > 	... 40 more
   [junit4]   2> 909337 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSingleFieldDiffAnalyzers
   [junit4]   2> 909337 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 909337 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:*} 0 0
   [junit4]   2> 909338 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909338 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909354 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2c82a026[collection1] main]
   [junit4]   2> 909355 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909355 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2c82a026[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):C1)))}
   [junit4]   2> 909363 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[1],commit=} 0 25
   [junit4]   2> 909364 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909364 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909364 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3fed8f4d[collection1] main]
   [junit4]   2> 909365 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909365 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3fed8f4d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):C1) Uninverting(_6(6.7.0):C1)))}
   [junit4]   2> 909365 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[2],commit=} 0 1
   [junit4]   2> 909365 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909365 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909366 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2141f9c1[collection1] main]
   [junit4]   2> 909366 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909366 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2141f9c1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):C1) Uninverting(_6(6.7.0):C1) Uninverting(_7(6.7.0):C1)))}
   [junit4]   2> 909367 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[3],commit=} 0 1
   [junit4]   2> 909367 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909367 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909368 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@631c6355[collection1] main]
   [junit4]   2> 909368 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909368 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@631c6355[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):C1) Uninverting(_6(6.7.0):C1) Uninverting(_7(6.7.0):C1) Uninverting(_8(6.7.0):C1)))}
   [junit4]   2> 909368 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[4],commit=} 0 1
   [junit4]   2> 909369 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909369 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@74cd4ac7
   [junit4]   2> 909370 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@6f7af9f9[collection1] main]
   [junit4]   2> 909370 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909370 INFO  (searcherExecutor-4108-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6f7af9f9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):C1) Uninverting(_6(6.7.0):C1) Uninverting(_7(6.7.0):C1) Uninverting(_8(6.7.0):C1) Uninverting(_9(6.7.0):C1)))}
   [junit4]   2> 909370 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[5],commit=} 0 1
   [junit4]   2> 909370 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 909370 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 909370 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 909370 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 0
   [junit4]   2> 909371 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=*:*&wt=xml} hits=5 status=0 QTime=0
   [junit4]   2> 909371 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter:let&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909372 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter:lett&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 909372 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:low&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909372 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:l&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 909372 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:lo&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 909372 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:lower&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 909373 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:whi&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909373 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:teS&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909373 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:in&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909373 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:white&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 909373 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:uni&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909374 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:cod&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909374 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:e&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909374 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:unico&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 909374 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=keyword:keyword&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 909374 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=keyword:key&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 909375 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[3543ED238D88DAA3]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSingleFieldDiffAnalyzers
   [junit4]   2> 909375 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 909375 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1154408032
   [junit4]   2> 909375 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 909376 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.jvm
   [junit4]   2> 909376 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.jetty
   [junit4]   2> 909388 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 909460 INFO  (coreCloseExecutor-4112-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@779cdd22
   [junit4]   2> 909464 INFO  (coreCloseExecutor-4112-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.util.TestMaxTokenLenTokenizer_3543ED238D88DAA3-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=ga, timezone=Asia/Irkutsk
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=3,threads=1,free=155906616,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrCloudWithDelegationTokens, QueryParsingTest, CoreAdminCreateDiscoverTest, BinaryUpdateRequestHandlerTest, TestConfig, TestExpandComponent, TestPushWriter, TestFieldCache, ExternalFileFieldSortTest, SpatialHeatmapFacetsTest, TestUninvertingReader, JsonLoaderTest, CdcrReplicationHandlerTest, TestSolr4Spatial, AutoCommitTest, CdcrVersionReplicationTest, TestRawTransformer, ZkStateWriterTest, DistributedQueryComponentOptimizationTest, LeaderInitiatedRecoveryOnShardRestartTest, SpellCheckComponentTest, TestConfigSetsAPI, BJQParserTest, TestSolrConfigHandler, AtomicUpdatesTest, ShardSplitTest, PreAnalyzedFieldManagedSchemaCloudTest, TestCollationFieldDocValues, TestClassicSimilarityFactory, StatsComponentTest, TestExactSharedStatsCache, ConfigSetsAPITest, TestCloudDeleteByQuery, SolrCmdDistributorTest, TestManagedStopFilterFactory, TestCoreDiscovery, BooleanFieldTest, TestUnifiedSolrHighlighter, TestTrie, DistribCursorPagingTest, FullSolrCloudDistribCmdsTest, TestCoreAdminApis, HdfsWriteToMultipleCollectionsTest, TestStressInPlaceUpdates, MissingSegmentRecoveryTest, TestManagedSchema, WordBreakSolrSpellCheckerTest, HdfsTlogReplayBufferedWhileIndexingTest, RequestHandlersTest, TestManagedSynonymGraphFilterFactory, DefaultValueUpdateProcessorTest, FieldMutatingUpdateProcessorTest, BasicFunctionalityTest, TestRTimerTree, TestNumericTerms32, BlockCacheTest, TestTolerantUpdateProcessorCloud, DistributedIntervalFacetingTest, TestRequestForwarding, TestComplexPhraseQParserPlugin, TestManagedSchemaThreadSafety, SliceStateTest, TestSolrQueryParser, TestSolrCoreProperties, TestConfigOverlay, TestGraphMLResponseWriter, URLClassifyProcessorTest, SolrGangliaReporterTest, CircularListTest, TestInPlaceUpdatesDistrib, TestRecoveryHdfs, TestDelegationWithHadoopAuth, OpenCloseCoreStressTest, TestStandardQParsers, RuleEngineTest, HighlighterConfigTest, TestFieldCacheSortRandom, MinimalSchemaTest, TestSuggestSpellingConverter, CollectionReloadTest, JavabinLoaderTest, TestMacroExpander, TestReqParamsAPI, ShardRoutingCustomTest, DeleteNodeTest, TestAuthorizationFramework, TestRuleBasedAuthorizationPlugin, TestNestedDocsSort, TestBadConfig, HdfsSyncSliceTest, TestValueSourceCache, RestartWhileUpdatingTest, RegexBoostProcessorTest, TestMergePolicyConfig, SolrInfoMBeanTest, ExitableDirectoryReaderTest, ZkSolrClientTest, TestDownShardTolerantSearch, SolrIndexSplitterTest, TestMissingGroups, TestAtomicUpdateErrorCases, TestMaxTokenLenTokenizer]
   [junit4] Completed [203/715 (1!)] on J1 in 0.46s, 2 tests, 1 error <<< FAILURES!

[...truncated 1116 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestDeleteCollectionOnDownNodes
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/init-core-data-001
   [junit4]   2> 2122054 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 2122060 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2122061 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001
   [junit4]   2> 2122061 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2122061 INFO  (Thread-4148) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2122061 INFO  (Thread-4148) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2122064 ERROR (Thread-4148) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2122163 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:52801
   [junit4]   2> 2122171 INFO  (jetty-launcher-3124-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2122171 INFO  (jetty-launcher-3124-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2122172 INFO  (jetty-launcher-3124-thread-3) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2122173 INFO  (jetty-launcher-3124-thread-4) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2122175 INFO  (jetty-launcher-3124-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@48420757{/solr,null,AVAILABLE}
   [junit4]   2> 2122176 INFO  (jetty-launcher-3124-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@cfd41bd{HTTP/1.1,[http/1.1]}{127.0.0.1:52803}
   [junit4]   2> 2122176 INFO  (jetty-launcher-3124-thread-1) [    ] o.e.j.s.Server Started @2127639ms
   [junit4]   2> 2122176 INFO  (jetty-launcher-3124-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52803}
   [junit4]   2> 2122176 ERROR (jetty-launcher-3124-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2122177 INFO  (jetty-launcher-3124-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.7.0
   [junit4]   2> 2122177 INFO  (jetty-launcher-3124-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2122177 INFO  (jetty-launcher-3124-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2122177 INFO  (jetty-launcher-3124-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-05-29T15:34:33.386Z
   [junit4]   2> 2122181 INFO  (jetty-launcher-3124-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2612a664{/solr,null,AVAILABLE}
   [junit4]   2> 2122181 INFO  (jetty-launcher-3124-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@16c13d52{HTTP/1.1,[http/1.1]}{127.0.0.1:52804}
   [junit4]   2> 2122181 INFO  (jetty-launcher-3124-thread-4) [    ] o.e.j.s.Server Started @2127644ms
   [junit4]   2> 2122181 INFO  (jetty-launcher-3124-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52804}
   [junit4]   2> 2122181 ERROR (jetty-launcher-3124-thread-4) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2122182 INFO  (jetty-launcher-3124-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.7.0
   [junit4]   2> 2122183 INFO  (jetty-launcher-3124-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2122183 INFO  (jetty-launcher-3124-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2122183 INFO  (jetty-launcher-3124-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-05-29T15:34:33.392Z
   [junit4]   2> 2122185 INFO  (jetty-launcher-3124-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@434366a6{/solr,null,AVAILABLE}
   [junit4]   2> 2122186 INFO  (jetty-launcher-3124-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1fd7c490{HTTP/1.1,[http/1.1]}{127.0.0.1:52806}
   [junit4]   2> 2122186 INFO  (jetty-launcher-3124-thread-3) [    ] o.e.j.s.Server Started @2127649ms
   [junit4]   2> 2122186 INFO  (jetty-launcher-3124-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52806}
   [junit4]   2> 2122186 ERROR (jetty-launcher-3124-thread-3) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2122187 INFO  (jetty-launcher-3124-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.7.0
   [junit4]   2> 2122187 INFO  (jetty-launcher-3124-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2122187 INFO  (jetty-launcher-3124-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2122187 INFO  (jetty-launcher-3124-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-05-29T15:34:33.396Z
   [junit4]   2> 2122189 INFO  (jetty-launcher-3124-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@55319951{/solr,null,AVAILABLE}
   [junit4]   2> 2122189 INFO  (jetty-launcher-3124-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5dc4dcd6{HTTP/1.1,[http/1.1]}{127.0.0.1:52809}
   [junit4]   2> 2122189 INFO  (jetty-launcher-3124-thread-2) [    ] o.e.j.s.Server Started @2127652ms
   [junit4]   2> 2122189 INFO  (jetty-launcher-3124-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=52809}
   [junit4]   2> 2122189 ERROR (jetty-launcher-3124-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2122190 INFO  (jetty-launcher-3124-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.7.0
   [junit4]   2> 2122190 INFO  (jetty-launcher-3124-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2122190 INFO  (jetty-launcher-3124-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2122190 INFO  (jetty-launcher-3124-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-05-29T15:34:33.399Z
   [junit4]   2> 2122191 INFO  (jetty-launcher-3124-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2122194 INFO  (jetty-launcher-3124-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2122201 INFO  (jetty-launcher-3124-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2122202 INFO  (jetty-launcher-3124-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2122205 INFO  (jetty-launcher-3124-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2122206 INFO  (jetty-launcher-3124-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52801/solr
   [junit4]   2> 2122209 INFO  (jetty-launcher-3124-thread-4) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2122210 INFO  (jetty-launcher-3124-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52801/solr
   [junit4]   2> 2122213 INFO  (jetty-launcher-3124-thread-3) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2122214 INFO  (jetty-launcher-3124-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2122215 INFO  (jetty-launcher-3124-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52801/solr
   [junit4]   2> 2122216 INFO  (jetty-launcher-3124-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52801/solr
   [junit4]   2> 2122310 INFO  (jetty-launcher-3124-thread-1) [n:127.0.0.1:52803_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2122310 INFO  (jetty-launcher-3124-thread-2) [n:127.0.0.1:52809_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2122311 INFO  (jetty-launcher-3124-thread-4) [n:127.0.0.1:52804_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2122312 INFO  (jetty-launcher-3124-thread-3) [n:127.0.0.1:52806_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2122314 INFO  (jetty-launcher-3124-thread-1) [n:127.0.0.1:52803_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52803_solr
   [junit4]   2> 2122315 INFO  (jetty-launcher-3124-thread-2) [n:127.0.0.1:52809_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52809_solr
   [junit4]   2> 2122318 INFO  (jetty-launcher-3124-thread-4) [n:127.0.0.1:52804_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52804_solr
   [junit4]   2> 2122318 INFO  (jetty-launcher-3124-thread-3) [n:127.0.0.1:52806_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52806_solr
   [junit4]   2> 2122318 INFO  (jetty-launcher-3124-thread-1) [n:127.0.0.1:52803_solr    ] o.a.s.c.Overseer Overseer (id=98046579393953801-127.0.0.1:52803_solr-n_0000000000) starting
   [junit4]   2> 2122321 INFO  (zkCallback-3144-thread-1-processing-n:127.0.0.1:52804_solr) [n:127.0.0.1:52804_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2122321 INFO  (zkCallback-3145-thread-1-processing-n:127.0.0.1:52806_solr) [n:127.0.0.1:52806_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2122322 INFO  (zkCallback-3146-thread-1-processing-n:127.0.0.1:52809_solr) [n:127.0.0.1:52809_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2122324 INFO  (zkCallback-3143-thread-1-processing-n:127.0.0.1:52803_solr) [n:127.0.0.1:52803_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2122325 INFO  (zkCallback-3144-thread-1-processing-n:127.0.0.1:52804_solr) [n:127.0.0.1:52804_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2122326 INFO  (zkCallback-3146-thread-1-processing-n:127.0.0.1:52809_solr) [n:127.0.0.1:52809_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2122326 INFO  (zkCallback-3143-thread-1-processing-n:127.0.0.1:52803_solr) [n:127.0.0.1:52803_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2122328 INFO  (zkCallback-3145-thread-1-processing-n:127.0.0.1:52806_solr) [n:127.0.0.1:52806_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
   [junit4]   2> 2122329 INFO  (zkCallback-3144-thread-1-processing-n:127.0.0.1:52804_solr) [n:127.0.0.1:52804_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2122330 INFO  (zkCallback-3143-thread-1-processing-n:127.0.0.1:52803_solr) [n:127.0.0.1:52803_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2122330 INFO  (zkCallback-3146-thread-1-processing-n:127.0.0.1:52809_solr) [n:127.0.0.1:52809_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2122377 INFO  (jetty-launcher-3124-thread-1) [n:127.0.0.1:52803_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52803_solr
   [junit4]   2> 2122381 INFO  (zkCallback-3146-thread-1-processing-n:127.0.0.1:52809_solr) [n:127.0.0.1:52809_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2122381 INFO  (zkCallback-3144-thread-1-processing-n:127.0.0.1:52804_solr) [n:127.0.0.1:52804_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2122382 INFO  (zkCallback-3145-thread-1-processing-n:127.0.0.1:52806_solr) [n:127.0.0.1:52806_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2122382 INFO  (zkCallback-3143-thread-1-processing-n:127.0.0.1:52803_solr) [n:127.0.0.1:52803_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2122484 INFO  (jetty-launcher-3124-thread-4) [n:127.0.0.1:52804_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node4/.
   [junit4]   2> 2122493 INFO  (jetty-launcher-3124-thread-2) [n:127.0.0.1:52809_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node2/.
   [junit4]   2> 2122493 INFO  (jetty-launcher-3124-thread-1) [n:127.0.0.1:52803_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node1/.
   [junit4]   2> 2122499 INFO  (jetty-launcher-3124-thread-3) [n:127.0.0.1:52806_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node3/.
   [junit4]   2> 2122517 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2122519 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52801/solr ready
   [junit4]   2> 2122565 INFO  (TEST-TestDeleteCollectionOnDownNodes.deleteCollectionWithDownNodes-seed#[3543ED238D88DAA3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting deleteCollectionWithDownNodes
   [junit4]   2> 2122570 INFO  (qtp1449247367-18409) [n:127.0.0.1:52809_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=3&collection.configName=conf&maxShardsPerNode=3&name=halfdeletedcollection2&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2122575 INFO  (OverseerThreadFactory-8632-thread-1-processing-n:127.0.0.1:52803_solr) [n:127.0.0.1:52803_solr    ] o.a.s.c.CreateCollectionCmd Create collection halfdeletedcollection2
   [junit4]   2> 2122691 INFO  (qtp401161163-18418) [n:127.0.0.1:52804_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard3_replica1&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard3&wt=javabin&version=2
   [junit4]   2> 2122691 INFO  (qtp1449247367-18410) [n:127.0.0.1:52809_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard2_replica1&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2122692 INFO  (qtp1449247367-18410) [n:127.0.0.1:52809_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2122692 INFO  (qtp401161163-18418) [n:127.0.0.1:52804_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2122693 INFO  (qtp1449247367-18412) [n:127.0.0.1:52809_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard4_replica3&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard4&wt=javabin&version=2
   [junit4]   2> 2122694 INFO  (qtp623397724-18389) [n:127.0.0.1:52803_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard1_replica1&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2122695 INFO  (qtp1449247367-18413) [n:127.0.0.1:52809_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard3_replica2&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard3&wt=javabin&version=2
   [junit4]   2> 2122696 INFO  (qtp623397724-18389) [n:127.0.0.1:52803_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2122696 INFO  (qtp2131933445-18402) [n:127.0.0.1:52806_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard4_replica1&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard4&wt=javabin&version=2
   [junit4]   2> 2122696 INFO  (qtp2131933445-18401) [n:127.0.0.1:52806_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard1_replica2&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2122697 INFO  (qtp2131933445-18402) [n:127.0.0.1:52806_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2122699 INFO  (qtp623397724-18394) [n:127.0.0.1:52803_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard3_replica3&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard3&wt=javabin&version=2
   [junit4]   2> 2122699 INFO  (qtp2131933445-18403) [n:127.0.0.1:52806_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard2_replica3&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2122699 INFO  (qtp401161163-18422) [n:127.0.0.1:52804_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard4_replica2&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard4&wt=javabin&version=2
   [junit4]   2> 2122699 INFO  (qtp623397724-18392) [n:127.0.0.1:52803_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard2_replica2&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard2&wt=javabin&version=2
   [junit4]   2> 2122698 INFO  (qtp401161163-18420) [n:127.0.0.1:52804_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=halfdeletedcollection2_shard1_replica3&action=CREATE&numShards=4&collection=halfdeletedcollection2&shard=shard1&wt=javabin&version=2
   [junit4]   2> 2122843 INFO  (zkCallback-3146-thread-1-processing-n:127.0.0.1:52809_solr) [n:127.0.0.1:52809_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection2/state.json] for collection [halfdeletedcollection2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2122843 INFO  (zkCallback-3143-thread-1-processing-n:127.0.0.1:52803_solr) [n:127.0.0.1:52803_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection2/state.json] for collection [halfdeletedcollection2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2122843 INFO  (zkCallback-3145-thread-1-processing-n:127.0.0.1:52806_solr) [n:127.0.0.1:52806_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection2/state.json] for collection [halfdeletedcollection2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2122843 INFO  (zkCallback-3144-thread-1-processing-n:127.0.0.1:52804_solr) [n:127.0.0.1:52804_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/halfdeletedcollection2/state.json] for collection [halfdeletedcollection2] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2123759 INFO  (qtp401161163-18422) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123768 INFO  (qtp2131933445-18402) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123801 INFO  (qtp2131933445-18403) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123803 INFO  (qtp401161163-18420) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123810 INFO  (qtp2131933445-18401) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123820 INFO  (qtp401161163-18422) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica2] o.a.s.s.IndexSchema [halfdeletedcollection2_shard4_replica2] Schema name=minimal
   [junit4]   2> 2123820 INFO  (qtp2131933445-18402) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica1] o.a.s.s.IndexSchema [halfdeletedcollection2_shard4_replica1] Schema name=minimal
   [junit4]   2> 2123823 INFO  (qtp2131933445-18402) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2123823 INFO  (qtp2131933445-18402) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica1] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection2_shard4_replica1' using configuration from collection halfdeletedcollection2, trusted=true
   [junit4]   2> 2123824 INFO  (qtp2131933445-18402) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2123824 INFO  (qtp2131933445-18402) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica1] o.a.s.c.SolrCore [[halfdeletedcollection2_shard4_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node3/halfdeletedcollection2_shard4_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node3/./halfdeletedcollection2_shard4_replica1/data/]
   [junit4]   2> 2123824 INFO  (qtp401161163-18420) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica3] o.a.s.s.IndexSchema [halfdeletedcollection2_shard1_replica3] Schema name=minimal
   [junit4]   2> 2123861 INFO  (qtp623397724-18392) [n:127.0.0.1:52803_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123863 INFO  (qtp401161163-18420) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2123863 INFO  (qtp401161163-18420) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica3] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection2_shard1_replica3' using configuration from collection halfdeletedcollection2, trusted=true
   [junit4]   2> 2123864 INFO  (qtp401161163-18420) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2123864 INFO  (qtp401161163-18420) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica3] o.a.s.c.SolrCore [[halfdeletedcollection2_shard1_replica3] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node4/halfdeletedcollection2_shard1_replica3], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node4/./halfdeletedcollection2_shard1_replica3/data/]
   [junit4]   2> 2123866 INFO  (qtp401161163-18422) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2123866 INFO  (qtp401161163-18422) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica2] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection2_shard4_replica2' using configuration from collection halfdeletedcollection2, trusted=true
   [junit4]   2> 2123867 INFO  (qtp401161163-18422) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2123867 INFO  (qtp401161163-18422) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica2] o.a.s.c.SolrCore [[halfdeletedcollection2_shard4_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node4/halfdeletedcollection2_shard4_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node4/./halfdeletedcollection2_shard4_replica2/data/]
   [junit4]   2> 2123868 INFO  (qtp2131933445-18403) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica3] o.a.s.s.IndexSchema [halfdeletedcollection2_shard2_replica3] Schema name=minimal
   [junit4]   2> 2123873 INFO  (qtp1449247367-18410) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123874 INFO  (qtp1449247367-18412) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123875 INFO  (qtp2131933445-18403) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2123875 INFO  (qtp2131933445-18403) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica3] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection2_shard2_replica3' using configuration from collection halfdeletedcollection2, trusted=true
   [junit4]   2> 2123875 INFO  (qtp2131933445-18403) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2123876 INFO  (qtp2131933445-18403) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica3] o.a.s.c.SolrCore [[halfdeletedcollection2_shard2_replica3] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node3/halfdeletedcollection2_shard2_replica3], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node3/./halfdeletedcollection2_shard2_replica3/data/]
   [junit4]   2> 2123877 INFO  (qtp2131933445-18401) [n:127.0.0.1:52806_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica2] o.a.s.s.IndexSchema [halfdeletedcollection2_shard1_replica2] Schema name=minimal
   [junit4]   2> 2123879 INFO  (qtp623397724-18389) [n:127.0.0.1:52803_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123896 INFO  (qtp401161163-18418) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123907 INFO  (qtp1449247367-18413) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123907 INFO  (qtp623397724-18394) [n:127.0.0.1:52803_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 2123933 INFO  (qtp1449247367-18410) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica1] o.a.s.s.IndexSchema [halfdeletedcollection2_shard2_replica1] Schema name=minimal
   [junit4]   2> 2123934 INFO  (qtp1449247367-18412) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica3] o.a.s.s.IndexSchema [halfdeletedcollection2_shard4_replica3] Schema name=minimal
   [junit4]   2> 2123936 INFO  (qtp1449247367-18410) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2123937 INFO  (qtp1449247367-18410) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection2_shard2_replica1' using configuration from collection halfdeletedcollection2, trusted=true
   [junit4]   2> 2123938 INFO  (qtp1449247367-18410) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2123938 INFO  (qtp1449247367-18412) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2123938 INFO  (qtp1449247367-18410) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard2  x:halfdeletedcollection2_shard2_replica1] o.a.s.c.SolrCore [[halfdeletedcollection2_shard2_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node2/halfdeletedcollection2_shard2_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node2/./halfdeletedcollection2_shard2_replica1/data/]
   [junit4]   2> 2123938 INFO  (qtp1449247367-18412) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica3] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection2_shard4_replica3' using configuration from collection halfdeletedcollection2, trusted=true
   [junit4]   2> 2123939 INFO  (qtp1449247367-18412) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2123939 INFO  (qtp1449247367-18412) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard4  x:halfdeletedcollection2_shard4_replica3] o.a.s.c.SolrCore [[halfdeletedcollection2_shard4_replica3] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node2/halfdeletedcollection2_shard4_replica3], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node2/./halfdeletedcollection2_shard4_replica3/data/]
   [junit4]   2> 2123942 INFO  (qtp401161163-18418) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica1] o.a.s.s.IndexSchema [halfdeletedcollection2_shard3_replica1] Schema name=minimal
   [junit4]   2> 2123944 INFO  (qtp401161163-18418) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2123944 INFO  (qtp401161163-18418) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica1] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection2_shard3_replica1' using configuration from collection halfdeletedcollection2, trusted=true
   [junit4]   2> 2123944 INFO  (qtp401161163-18418) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2123944 INFO  (qtp401161163-18418) [n:127.0.0.1:52804_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica1] o.a.s.c.SolrCore [[halfdeletedcollection2_shard3_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node4/halfdeletedcollection2_shard3_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node4/./halfdeletedcollection2_shard3_replica1/data/]
   [junit4]   2> 2123945 INFO  (qtp1449247367-18413) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica2] o.a.s.s.IndexSchema [halfdeletedcollection2_shard3_replica2] Schema name=minimal
   [junit4]   2> 2123946 INFO  (qtp623397724-18389) [n:127.0.0.1:52803_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica1] o.a.s.s.IndexSchema [halfdeletedcollection2_shard1_replica1] Schema name=minimal
   [junit4]   2> 2123947 INFO  (qtp1449247367-18413) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2123947 INFO  (qtp1449247367-18413) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica2] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection2_shard3_replica2' using configuration from collection halfdeletedcollection2, trusted=true
   [junit4]   2> 2123948 INFO  (qtp1449247367-18413) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2123948 INFO  (qtp1449247367-18413) [n:127.0.0.1:52809_solr c:halfdeletedcollection2 s:shard3  x:halfdeletedcollection2_shard3_replica2] o.a.s.c.SolrCore [[halfdeletedcollection2_shard3_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node2/halfdeletedcollection2_shard3_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node2/./halfdeletedcollection2_shard3_replica2/data/]
   [junit4]   2> 2123948 INFO  (qtp623397724-18389) [n:127.0.0.1:52803_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2123949 INFO  (qtp623397724-18389) [n:127.0.0.1:52803_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'halfdeletedcollection2_shard1_replica1' using configuration from collection halfdeletedcollection2, trusted=true
   [junit4]   2> 2123949 INFO  (qtp623397724-18389) [n:127.0.0.1:52803_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2123949 INFO  (qtp623397724-18389) [n:127.0.0.1:52803_solr c:halfdeletedcollection2 s:shard1  x:halfdeletedcollection2_shard1_replica1] o.a.s.c.SolrCore [[halfdeletedcollection2_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001/tempDir-001/node1/halfdeletedcollection2_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.

[...truncated too long message...]

entCnxn.java:1406)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1097)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:323)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:320)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerNodePrioritizer.prioritizeOverseerNodes(OverseerNodePrioritizer.java:61)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:178)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2155084 WARN  (zkCallback-3145-thread-1-processing-n:127.0.0.1:52806_solr) [n:127.0.0.1:52806_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2155085 INFO  (jetty-closer-3125-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@434366a6{/solr,null,UNAVAILABLE}
   [junit4]   2> 2156595 WARN  (zkCallback-3144-thread-1-processing-n:127.0.0.1:52804_solr) [n:127.0.0.1:52804_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2156595 INFO  (jetty-closer-3125-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2612a664{/solr,null,UNAVAILABLE}
   [junit4]   2> 2156596 ERROR (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2156597 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52801 52801
   [junit4]   2> 2156698 INFO  (Thread-4148) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52801 52801
   [junit4]   2> 2156913 WARN  (Thread-4148) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/collections/halfdeletedcollection2/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/live_nodes
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 
   [junit4]   2> 2156914 INFO  (SUITE-TestDeleteCollectionOnDownNodes-seed#[3543ED238D88DAA3]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestDeleteCollectionOnDownNodes_3543ED238D88DAA3-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=1744, maxMBSortInHeap=5.471223574054843, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=es-MX, timezone=MST7MDT
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=3,threads=1,free=112190120,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestCollectionAPI, PingRequestHandlerTest, TestExtendedDismaxParser, SoftAutoCommitTest, ManagedSchemaRoundRobinCloudTest, PropertiesRequestHandlerTest, ActionThrottleTest, TestSimpleQParserPlugin, BlockJoinFacetSimpleTest, TestLeaderElectionWithEmptyReplica, TestXmlQParser, HighlighterTest, InfixSuggestersTest, TestHighFrequencyDictionaryFactory, TestOnReconnectListenerSupport, TestSolrFieldCacheMBean, DistributedQueryComponentCustomSortTest, TestSlowCompositeReaderWrapper, TestWordDelimiterFilterFactory, DocValuesMissingTest, DocExpirationUpdateProcessorFactoryTest, TestSolrDeletionPolicy1, DistributedSuggestComponentTest, AlternateDirectoryTest, UtilsToolTest, BasicDistributedZkTest, TestConfigSetImmutable, QueryEqualityTest, ReplicationFactorTest, TestSolrCloudSnapshots, SolrSlf4jReporterTest, TestDFISimilarityFactory, MetricUtilsTest, SmileWriterTest, TestHighlightDedupGrouping, BasicAuthIntegrationTest, TestJettySolrRunner, TestFieldTypeResource, TestConfigSets, TestFieldResource, ZkCLITest, TestCoreContainer, RankQueryTest, CollectionStateFormat2Test, UUIDUpdateProcessorFallbackTest, TestExactStatsCache, TestDynamicFieldResource, HdfsDirectoryFactoryTest, DocumentBuilderTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnCommitTest, SolrGraphiteReporterTest, TestStressCloudBlindAtomicUpdates, TestSchemalessBufferedUpdates, TestNamedUpdateProcessors, DistributedFacetPivotLongTailTest, TestIntervalFaceting, SolrMetricsIntegrationTest, TestLRUCache, ConnectionManagerTest, PKIAuthenticationIntegrationTest, RollingRestartTest, CustomCollectionTest, TestConfigReload, TestDocTermOrds, TestFilteredDocIdSet, TestOverriddenPrefixQueryForCustomFieldType, StandardRequestHandlerTest, TestUpdate, TestHdfsUpdateLog, CheckHdfsIndexTest, HdfsChaosMonkeyNothingIsSafeTest, TestTrackingShardHandlerFactory, HdfsUnloadDistributedZkTest, TestRemoteStreaming, TestAddFieldRealTimeGet, TestConfigSetsAPIExclusivity, PathHierarchyTokenizerFactoryTest, OutputWriterTest, CdcrRequestHandlerTest, HdfsThreadLeakTest, ShufflingReplicaListTransformerTest, SubstringBytesRefFilterTest, PeerSyncTest, SolrPluginUtilsTest, TestCloudRecovery, ParsingFieldUpdateProcessorsTest, DistributedSpellCheckComponentTest, TestRawResponseWriter, HttpPartitionTest, ClassificationUpdateProcessorTest, TestMultiWordSynonyms, TestImpersonationWithHadoopAuth, DeleteStatusTest, TestFaceting, TestReplicationHandler, RulesTest, TestEmbeddedSolrServerConstructors, TestDocumentBuilder, TestUniqueKeyFieldResource, TestCustomDocTransformer, TestLockTree, ConfigureRecoveryStrategyTest, BadCopyFieldTest, IndexBasedSpellCheckerTest, V2ApiIntegrationTest, DeleteReplicaTest, SaslZkACLProviderTest, TestRestoreCore, PluginInfoTest, ZkControllerTest, TestBinaryResponseWriter, TestManagedResourceStorage, TestDistribIDF, DistributedFacetExistsSmallTest, TestFileDictionaryLookup, SegmentsInfoRequestHandlerTest, TestStressVersions, TestDistributedSearch, HdfsCollectionsAPIDistributedZkTest, AnalysisAfterCoreReloadTest, TestPointFields, MigrateRouteKeyTest, TestCloudInspectUtil, TestRandomRequestDistribution, SpatialFilterTest, SyncSliceTest, TestHttpShardHandlerFactory, TestJavabinTupleStreamParser, DocValuesNotIndexedTest, TestCSVResponseWriter, TestExclusionRuleCollectionAccess, TestGroupingSearch, TimeZoneUtilsTest, TestCryptoKeys, TestHalfAndHalfDocValues, TestHashPartitioner, BitVectorTest, SchemaVersionSpecificBehaviorTest, SimpleMLTQParserTest, TestDistributedStatsComponentCardinality, TestSweetSpotSimilarityFactory, TestDocBasedVersionConstraints, BadComponentTest, SuggestComponentContextFilterQueryTest, TestSolrIndexConfig, TestJsonFacetRefinement, TestBackupRepositoryFactory, TestJsonRequest, TestReloadDeadlock, TestObjectReleaseTracker, TestStressReorder, ConjunctionSolrSpellCheckerTest, HighlighterMaxOffsetTest, MoveReplicaTest, TestMacros, FastVectorHighlighterTest, TestSolrJ, TestSolrDynamicMBean, JvmMetricsTest, HdfsRecoverLeaseTest, AtomicUpdateProcessorFactoryTest, CloudExitableDirectoryReaderTest, TestSolrCLIRunExample, NodeMutatorTest, DistributedMLTComponentTest, TestFieldCacheReopen, TestSolrConfigHandlerConcurrent, InfoHandlerTest, SolrCoreTest, ZkNodePropsTest, CollectionsAPIAsyncDistributedZkTest, DocumentAnalysisRequestHandlerTest, TestStressUserVersions, OpenExchangeRatesOrgProviderTest, BasicZkTest, RecoveryZkTest, ClusterStateUpdateTest, TestRandomDVFaceting, TestRealTimeGet, TestJoin, HardAutoCommitTest, TestRangeQuery, SimpleFacetsTest, QueryElevationComponentTest, TestSort, TestFiltering, TestLazyCores, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, DocValuesMultiTest, DistanceFunctionTest, DebugComponentTest, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, TestOmitPositions, TermsComponentTest, TermVectorComponentTest, RequiredFieldsTest, IndexSchemaRuntimeFieldTest, TestJmxIntegration, UpdateRequestProcessorFactoryTest, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, CSVRequestHandlerTest, SearchHandlerTest, SOLR749Test, TestQuerySenderListener, UpdateParamsTest, TestQuerySenderNoQuery, CopyFieldTest, TestStressRecovery, PreAnalyzedFieldTest, PrimUtilsTest, DateFieldTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestUtils, SystemInfoHandlerTest, UUIDFieldTest, FileUtilsTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestTolerantSearch, TestPathTrie, AssignTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CollectionsAPISolrJTest, DistribJoinFromCollectionTest, DistributedQueueTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, PeerSyncReplicationTest, RecoveryAfterSoftCommitTest, RemoteQueryErrorTest, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestCloudPseudoReturnFields, TestClusterProperties, TestDeleteCollectionOnDownNodes]
   [junit4] Completed [548/715 (2!)] on J0 in 34.87s, 1 test, 1 failure <<< FAILURES!

[...truncated 44045 lines...]

[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 904 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/904/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([194B999CBFDDAFD1:7399A6F3E73E7F1E]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:895)
	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=letter0:lett&wt=xml
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
	... 40 more




Build Log:
[...truncated 12024 lines...]
   [junit4] Suite: org.apache.solr.util.TestMaxTokenLenTokenizer
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.util.TestMaxTokenLenTokenizer_194B999CBFDDAFD1-001/init-core-data-001
   [junit4]   2> 1398942 WARN  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1398943 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 1398945 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1398946 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 1398946 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1398976 WARN  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1398977 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1398991 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 1398997 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1399004 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 1399071 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48173c94
   [junit4]   2> 1399073 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48173c94
   [junit4]   2> 1399074 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48173c94
   [junit4]   2> 1399075 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1399124 WARN  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1399125 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1399157 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1399164 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1399164 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 1399165 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48173c94
   [junit4]   2> 1399165 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1399165 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.util.TestMaxTokenLenTokenizer_194B999CBFDDAFD1-001/init-core-data-001/]
   [junit4]   2> 1399166 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1399219 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1399220 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1399220 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=69.4072265625, floorSegmentMB=2.1064453125, forceMergeDeletesPctAllowed=26.640873008657632, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1399220 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e149c8b[collection1] main]
   [junit4]   2> 1399220 WARN  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 1399221 INFO  (coreLoadExecutor-6372-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1399221 INFO  (searcherExecutor-6373-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5e149c8b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1399222 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 1399223 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSingleFieldDiffAnalyzers
   [junit4]   2> 1399224 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 1399224 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:*} 0 0
   [junit4]   2> 1399224 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399224 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399226 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@304b3b36[collection1] main]
   [junit4]   2> 1399226 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399226 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@304b3b36[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1)))}
   [junit4]   2> 1399226 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[1],commit=} 0 2
   [junit4]   2> 1399227 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399227 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399228 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@40c1645d[collection1] main]
   [junit4]   2> 1399228 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399228 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@40c1645d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1) Uninverting(_1(6.7.0):c1)))}
   [junit4]   2> 1399228 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[2],commit=} 0 1
   [junit4]   2> 1399229 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399229 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399230 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2f4295fa[collection1] main]
   [junit4]   2> 1399230 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399230 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2f4295fa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1) Uninverting(_1(6.7.0):c1) Uninverting(_2(6.7.0):c1)))}
   [junit4]   2> 1399230 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[3],commit=} 0 1
   [junit4]   2> 1399231 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399231 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399231 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@f1483d4[collection1] main]
   [junit4]   2> 1399232 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399232 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@f1483d4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1) Uninverting(_1(6.7.0):c1) Uninverting(_2(6.7.0):c1) Uninverting(_3(6.7.0):c1)))}
   [junit4]   2> 1399232 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[4],commit=} 0 1
   [junit4]   2> 1399232 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399232 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399233 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@39cb2f60[collection1] main]
   [junit4]   2> 1399233 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399234 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@39cb2f60[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1) Uninverting(_1(6.7.0):c1) Uninverting(_2(6.7.0):c1) Uninverting(_3(6.7.0):c1) Uninverting(_4(6.7.0):c1)))}
   [junit4]   2> 1399234 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[5],commit=} 0 1
   [junit4]   2> 1399234 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399234 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1399234 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399234 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 0
   [junit4]   2> 1399237 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=*:*&wt=xml} hits=5 status=0 QTime=0
   [junit4]   2> 1399238 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter:let&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399238 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter:lett&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1399238 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:low&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399238 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:l&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1399238 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:lo&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1399239 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:lower&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1399239 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:whi&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399239 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:teS&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399239 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:in&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399239 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:white&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1399239 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:uni&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399240 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:cod&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399240 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:e&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399240 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:unico&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1399240 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=keyword:keyword&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399240 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=keyword:key&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1399240 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSingleFieldDiffAnalyzers
   [junit4]   2> 1399242 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSingleFieldSameAnalyzers
   [junit4]   2> 1399242 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 1399242 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:*} 0 0
   [junit4]   2> 1399243 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399243 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399243 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@6682ad6a[collection1] main]
   [junit4]   2> 1399244 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399244 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6682ad6a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1)))}
   [junit4]   2> 1399244 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[1],commit=} 0 1
   [junit4]   2> 1399244 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399244 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399245 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@47117856[collection1] main]
   [junit4]   2> 1399245 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399246 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@47117856[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1) Uninverting(_6(6.7.0):c1)))}
   [junit4]   2> 1399246 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[2],commit=} 0 1
   [junit4]   2> 1399246 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399246 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399247 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@47ecbd3d[collection1] main]
   [junit4]   2> 1399247 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399247 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@47ecbd3d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1) Uninverting(_6(6.7.0):c1) Uninverting(_7(6.7.0):c1)))}
   [junit4]   2> 1399247 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[3],commit=} 0 1
   [junit4]   2> 1399248 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399248 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399249 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e46a1ee[collection1] main]
   [junit4]   2> 1399249 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399249 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5e46a1ee[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1) Uninverting(_6(6.7.0):c1) Uninverting(_7(6.7.0):c1) Uninverting(_8(6.7.0):c1)))}
   [junit4]   2> 1399249 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[4],commit=} 0 1
   [junit4]   2> 1399249 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399249 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@56ab0fba
   [junit4]   2> 1399250 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3446f207[collection1] main]
   [junit4]   2> 1399251 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399251 INFO  (searcherExecutor-6373-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3446f207[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1) Uninverting(_6(6.7.0):c1) Uninverting(_7(6.7.0):c1) Uninverting(_8(6.7.0):c1) Uninverting(_9(6.7.0):c1)))}
   [junit4]   2> 1399251 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[5],commit=} 0 1
   [junit4]   2> 1399251 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1399251 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1399251 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1399251 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 0
   [junit4]   2> 1399251 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=*:*&wt=xml} hits=5 status=0 QTime=0
   [junit4]   2> 1399252 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter0:l&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1399252 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter0:let&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 1399253 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter0:lett&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 1399253 ERROR (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=letter0:lett&wt=xml
   [junit4]   2> 1399253 ERROR (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=letter0:lett&wt=xml:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=letter0:lett&wt=xml
   [junit4]   2> 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
   [junit4]   2> 	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 1399253 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[194B999CBFDDAFD1]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSingleFieldSameAnalyzers
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestMaxTokenLenTokenizer -Dtests.method=testSingleFieldSameAnalyzers -Dtests.seed=194B999CBFDDAFD1 -Dtests.slow=true -Dtests.locale=hu -Dtests.timezone=Asia/Ust-Nera -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.01s J1 | TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during query
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([194B999CBFDDAFD1:7399A6F3E73E7F1E]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:895)
   [junit4]    > 	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]    > 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]    > </response>
   [junit4]    > 	request was:q=letter0:lett&wt=xml
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
   [junit4]    > 	... 40 more
   [junit4]   2> 1399255 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 1399255 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=178994930
   [junit4]   2> 1399255 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 1399256 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.jvm
   [junit4]   2> 1399256 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.jetty
   [junit4]   2> 1399257 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[194B999CBFDDAFD1]-worker) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1399257 INFO  (coreCloseExecutor-6377-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@47260ca5
   [junit4]   2> 1399260 INFO  (coreCloseExecutor-6377-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.util.TestMaxTokenLenTokenizer_194B999CBFDDAFD1-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {lowerCase0=PostingsFormat(name=LuceneVarGapFixedInterval), whiteSpace=PostingsFormat(name=LuceneFixedGap), letter=PostingsFormat(name=Memory doPackFST= true), lowerCase=PostingsFormat(name=LuceneVarGapFixedInterval), unicodeWhiteSpace=PostingsFormat(name=Memory doPackFST= true), letter0=PostingsFormat(name=Memory doPackFST= true), unicodeWhiteSpace0=PostingsFormat(name=Memory doPackFST= true), keyword0=PostingsFormat(name=LuceneVarGapFixedInterval), id=PostingsFormat(name=LuceneVarGapFixedInterval), keyword=PostingsFormat(name=LuceneVarGapFixedInterval), whiteSpace0=PostingsFormat(name=LuceneFixedGap)}, docValues:{}, maxPointsInLeafNode=883, maxMBSortInHeap=5.3355181742144495, sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=hu, timezone=Asia/Ust-Nera
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=3,threads=1,free=149222832,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestExportWriter, TestCoreAdminApis, TestClusterProperties, CollectionReloadTest, SparseHLLTest, OpenCloseCoreStressTest, CoreMergeIndexesAdminHandlerTest, SolrIndexMetricsTest, TestLuceneMatchVersion, ZkSolrClientTest, SpellPossibilityIteratorTest, TestPHPSerializedResponseWriter, SpellCheckCollatorTest, CloudMLTQParserTest, ReplicaListTransformerTest, EnumFieldTest, TestAuthorizationFramework, TestChildDocTransformer, TestReload, ReturnFieldsTest, TestNestedDocsSort, BJQParserTest, TestTolerantUpdateProcessorCloud, QueryResultKeyTest, BasicFunctionalityTest, TestPathTrie, TestHdfsBackupRestoreCore, HdfsChaosMonkeySafeLeaderTest, DistributedTermsComponentTest, BigEndianAscendingWordSerializerTest, TestCollationFieldDocValues, TestSearcherReuse, FacetPivotSmallTest, DistributedIntervalFacetingTest, HdfsRecoveryZkTest, DistributedFacetPivotLargeTest, TestSchemaSimilarityResource, SuggesterWFSTTest, TestFiltering, TestRecoveryHdfs, ShowFileRequestHandlerTest, TestRealTimeGet, TermVectorComponentTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestExactSharedStatsCache, TestSolrQueryParserResource, SortByFunctionTest, CoreAdminHandlerTest, CollectionTooManyReplicasTest, RegexBoostProcessorTest, TestSolrCloudWithHadoopAuthPlugin, SliceStateTest, CleanupOldIndexTest, ExplicitHLLTest, SolrJmxReporterTest, StatsReloadRaceTest, TestFuzzyAnalyzedSuggestions, TestLocalFSCloudBackupRestore, TestFieldCacheSortRandom, TestPayloadCheckQParserPlugin, TestFieldCache, TestSolrConfigHandler, TestInPlaceUpdatesDistrib, TestTrieFacet, RequestHandlersTest, SolrXmlInZkTest, TestCorePropertiesReload, UpdateLogTest, MissingSegmentRecoveryTest, TestHdfsCloudBackupRestore, ShardRoutingCustomTest, TestIndexingPerformance, OverseerRolesTest, TestQueryTypes, TestSubQueryTransformerCrossCore, MergeStrategyTest, SolrCoreCheckLockOnStartupTest, TermVectorComponentDistributedTest, LeaderElectionContextKeyTest, SecurityConfHandlerTest, TestEmbeddedSolrServerSchemaAPI, FieldMutatingUpdateProcessorTest, DebugComponentTest, CdcrVersionReplicationTest, TestRebalanceLeaders, TestQueryUtils, PreAnalyzedFieldTest, TestManagedSchema, NotRequiredUniqueKeyTest, TestManagedSynonymGraphFilterFactory, MinimalSchemaTest, TestSolrCloudWithSecureImpersonation, TestConfigSetsAPI, TestCollectionAPIs, TestDynamicFieldCollectionResource, TestLMDirichletSimilarityFactory, SuggesterTSTTest, TestSSLRandomization, DeleteShardTest, TestMiniSolrCloudCluster, TestReqParamsAPI, TestClassNameShortening, TestSolrCoreProperties, SuggesterTest, TestRequestForwarding, MBeansHandlerTest, TestPushWriter, TestSchemaManager, TestSystemIdResolver, TestBadConfig, DistributedFacetPivotWhiteBoxTest, TestRecovery, ConvertedLegacyTest, TestCloudPseudoReturnFields, TestConfig, BasicZkTest, LeaderElectionIntegrationTest, TestRestManager, TestSearchPerf, TestUninvertingReader, FullSolrCloudDistribCmdsTest, DataDrivenBlockJoinTest, LeaderFailureAfterFreshStartTest, TestNonDefinedSimilarityFactory, TestDistributedMissingSort, CoreSorterTest, DistributedFacetPivotSmallAdvancedTest, TestJsonFacets, DistribCursorPagingTest, ChaosMonkeySafeLeaderTest, TestSolrCoreParser, DeleteNodeTest, SpellCheckComponentTest, SuggestComponentTest, TestLeaderElectionZkExpiry, TestSmileRequest, FileBasedSpellCheckerTest, JavabinLoaderTest, DefaultValueUpdateProcessorTest, AnalysisErrorHandlingTest, SSLMigrationTest, ZkStateWriterTest, ScriptEngineTest, TestScoreJoinQPScore, RecoveryAfterSoftCommitTest, MetricsHandlerTest, TestManagedResource, CursorMarkTest, TemplateUpdateProcessorTest, SolrCoreMetricManagerTest, TestLRUStatsCache, TestShortCircuitedRequests, ShardSplitTest, SpellCheckCollatorWithCollapseTest, SmileWriterTest, TestSerializedLuceneMatchVersion, TestZkChroot, TestFastWriter, TestRandomFaceting, TestCrossCoreJoin, SolrGangliaReporterTest, TestLMJelinekMercerSimilarityFactory, UpdateParamsTest, TestMaxTokenLenTokenizer]
   [junit4] Completed [341/715 (1!)] on J1 in 0.34s, 2 tests, 1 error <<< FAILURES!

[...truncated 44672 lines...]

[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 903 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/903/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
FAILED:  org.apache.solr.cloud.ReplaceNodeTest.test

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([1F5880993BA87A28:970CBF43955417D0]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.ReplaceNodeTest.test(ReplaceNodeTest.java:82)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([1F5880993BA87A28:758ABFF6634BAAE7]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:895)
	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=letter0:lett&wt=xml
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
	... 40 more




Build Log:
[...truncated 11995 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplaceNodeTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/init-core-data-001
   [junit4]   2> 1117669 WARN  (SUITE-ReplaceNodeTest-seed#[1F5880993BA87A28]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1117669 INFO  (SUITE-ReplaceNodeTest-seed#[1F5880993BA87A28]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 1117672 INFO  (SUITE-ReplaceNodeTest-seed#[1F5880993BA87A28]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1117672 INFO  (SUITE-ReplaceNodeTest-seed#[1F5880993BA87A28]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 6 servers in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001
   [junit4]   2> 1117673 INFO  (SUITE-ReplaceNodeTest-seed#[1F5880993BA87A28]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1117673 INFO  (Thread-3148) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1117673 INFO  (Thread-3148) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1117675 ERROR (Thread-3148) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1117779 INFO  (SUITE-ReplaceNodeTest-seed#[1F5880993BA87A28]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:65200
   [junit4]   2> 1117815 INFO  (jetty-launcher-3657-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1117815 INFO  (jetty-launcher-3657-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1117816 INFO  (jetty-launcher-3657-thread-3) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1117816 INFO  (jetty-launcher-3657-thread-4) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1117817 INFO  (jetty-launcher-3657-thread-5) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1117818 INFO  (jetty-launcher-3657-thread-6) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1117829 INFO  (jetty-launcher-3657-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@206a3ac4{/solr,null,AVAILABLE}
   [junit4]   2> 1117829 INFO  (jetty-launcher-3657-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e55e747{/solr,null,AVAILABLE}
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@43d59fb0{HTTP/1.1,[http/1.1]}{127.0.0.1:65202}
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-2) [    ] o.e.j.s.Server Started @1121223ms
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4da21b8c{HTTP/1.1,[http/1.1]}{127.0.0.1:65203}
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65202}
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-4) [    ] o.e.j.s.Server Started @1121223ms
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65203}
   [junit4]   2> 1117830 ERROR (jetty-launcher-3657-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-6) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7bdb8ae2{/solr,null,AVAILABLE}
   [junit4]   2> 1117830 ERROR (jetty-launcher-3657-thread-4) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 6.7.0
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 6.7.0
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1117830 INFO  (jetty-launcher-3657-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1117831 INFO  (jetty-launcher-3657-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1117831 INFO  (jetty-launcher-3657-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-06-02T12:15:04.961Z
   [junit4]   2> 1117831 INFO  (jetty-launcher-3657-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-06-02T12:15:04.961Z
   [junit4]   2> 1117831 INFO  (jetty-launcher-3657-thread-5) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3b5d424b{/solr,null,AVAILABLE}
   [junit4]   2> 1117831 INFO  (jetty-launcher-3657-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5fcbcfd7{/solr,null,AVAILABLE}
   [junit4]   2> 1117832 INFO  (jetty-launcher-3657-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1c40ea9c{HTTP/1.1,[http/1.1]}{127.0.0.1:65206}
   [junit4]   2> 1117832 INFO  (jetty-launcher-3657-thread-1) [    ] o.e.j.s.Server Started @1121225ms
   [junit4]   2> 1117832 INFO  (jetty-launcher-3657-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65206}
   [junit4]   2> 1117832 INFO  (jetty-launcher-3657-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@31d0a3d2{/solr,null,AVAILABLE}
   [junit4]   2> 1117832 ERROR (jetty-launcher-3657-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1117832 INFO  (jetty-launcher-3657-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@43f1bccb{HTTP/1.1,[http/1.1]}{127.0.0.1:65207}
   [junit4]   2> 1117832 INFO  (jetty-launcher-3657-thread-3) [    ] o.e.j.s.Server Started @1121225ms
   [junit4]   2> 1117832 INFO  (jetty-launcher-3657-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65207}
   [junit4]   2> 1117832 INFO  (jetty-launcher-3657-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 6.7.0
   [junit4]   2> 1117832 ERROR (jetty-launcher-3657-thread-3) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1117832 INFO  (jetty-launcher-3657-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1117833 INFO  (jetty-launcher-3657-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1117833 INFO  (jetty-launcher-3657-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 6.7.0
   [junit4]   2> 1117833 INFO  (jetty-launcher-3657-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-06-02T12:15:04.963Z
   [junit4]   2> 1117833 INFO  (jetty-launcher-3657-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1117833 INFO  (jetty-launcher-3657-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1117833 INFO  (jetty-launcher-3657-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-06-02T12:15:04.963Z
   [junit4]   2> 1117834 INFO  (jetty-launcher-3657-thread-5) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6c14f590{HTTP/1.1,[http/1.1]}{127.0.0.1:65205}
   [junit4]   2> 1117834 INFO  (jetty-launcher-3657-thread-5) [    ] o.e.j.s.Server Started @1121228ms
   [junit4]   2> 1117835 INFO  (jetty-launcher-3657-thread-5) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65205}
   [junit4]   2> 1117835 INFO  (jetty-launcher-3657-thread-6) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3f223a39{HTTP/1.1,[http/1.1]}{127.0.0.1:65204}
   [junit4]   2> 1117835 INFO  (jetty-launcher-3657-thread-6) [    ] o.e.j.s.Server Started @1121228ms
   [junit4]   2> 1117835 INFO  (jetty-launcher-3657-thread-6) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65204}
   [junit4]   2> 1117835 ERROR (jetty-launcher-3657-thread-6) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1117835 ERROR (jetty-launcher-3657-thread-5) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1117836 INFO  (jetty-launcher-3657-thread-6) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 6.7.0
   [junit4]   2> 1117836 INFO  (jetty-launcher-3657-thread-5) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 6.7.0
   [junit4]   2> 1117836 INFO  (jetty-launcher-3657-thread-6) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1117836 INFO  (jetty-launcher-3657-thread-5) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1117836 INFO  (jetty-launcher-3657-thread-6) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1117836 INFO  (jetty-launcher-3657-thread-5) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1117836 INFO  (jetty-launcher-3657-thread-6) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-06-02T12:15:04.966Z
   [junit4]   2> 1117836 INFO  (jetty-launcher-3657-thread-5) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-06-02T12:15:04.966Z
   [junit4]   2> 1117839 INFO  (jetty-launcher-3657-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1117839 INFO  (jetty-launcher-3657-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1117844 INFO  (jetty-launcher-3657-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1117844 INFO  (jetty-launcher-3657-thread-5) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1117848 INFO  (jetty-launcher-3657-thread-6) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1117849 INFO  (jetty-launcher-3657-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1117859 INFO  (jetty-launcher-3657-thread-4) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1117860 INFO  (jetty-launcher-3657-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1117860 INFO  (jetty-launcher-3657-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:65200/solr
   [junit4]   2> 1117861 INFO  (jetty-launcher-3657-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:65200/solr
   [junit4]   2> 1117862 INFO  (jetty-launcher-3657-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1117862 INFO  (jetty-launcher-3657-thread-5) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1117863 INFO  (jetty-launcher-3657-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:65200/solr
   [junit4]   2> 1117863 INFO  (jetty-launcher-3657-thread-5) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:65200/solr
   [junit4]   2> 1117864 INFO  (jetty-launcher-3657-thread-3) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1117864 INFO  (jetty-launcher-3657-thread-6) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1117864 INFO  (jetty-launcher-3657-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:65200/solr
   [junit4]   2> 1117871 INFO  (jetty-launcher-3657-thread-6) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:65200/solr
   [junit4]   2> 1117911 INFO  (jetty-launcher-3657-thread-5) [n:127.0.0.1:65205_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1117911 INFO  (jetty-launcher-3657-thread-5) [n:127.0.0.1:65205_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:65205_solr
   [junit4]   2> 1117913 INFO  (jetty-launcher-3657-thread-5) [n:127.0.0.1:65205_solr    ] o.a.s.c.Overseer Overseer (id=98068444270100497-127.0.0.1:65205_solr-n_0000000000) starting
   [junit4]   2> 1117923 INFO  (jetty-launcher-3657-thread-5) [n:127.0.0.1:65205_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65205_solr
   [junit4]   2> 1117925 INFO  (zkCallback-3688-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1117926 INFO  (zkCallback-3686-thread-1-processing-n:127.0.0.1:65207_solr) [n:127.0.0.1:65207_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1117926 INFO  (zkCallback-3687-thread-1-processing-n:127.0.0.1:65206_solr) [n:127.0.0.1:65206_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1117927 INFO  (zkCallback-3685-thread-1-processing-n:127.0.0.1:65203_solr) [n:127.0.0.1:65203_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1117928 INFO  (jetty-launcher-3657-thread-3) [n:127.0.0.1:65207_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1117929 INFO  (jetty-launcher-3657-thread-4) [n:127.0.0.1:65203_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1117931 INFO  (jetty-launcher-3657-thread-2) [n:127.0.0.1:65202_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1117932 INFO  (zkCallback-3684-thread-1-processing-n:127.0.0.1:65202_solr) [n:127.0.0.1:65202_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1117932 INFO  (jetty-launcher-3657-thread-1) [n:127.0.0.1:65206_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1117937 INFO  (jetty-launcher-3657-thread-2) [n:127.0.0.1:65202_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65202_solr
   [junit4]   2> 1117939 INFO  (jetty-launcher-3657-thread-4) [n:127.0.0.1:65203_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65203_solr
   [junit4]   2> 1117940 INFO  (zkCallback-3686-thread-1-processing-n:127.0.0.1:65207_solr) [n:127.0.0.1:65207_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1117940 INFO  (zkCallback-3687-thread-1-processing-n:127.0.0.1:65206_solr) [n:127.0.0.1:65206_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1117940 INFO  (zkCallback-3684-thread-1-processing-n:127.0.0.1:65202_solr) [n:127.0.0.1:65202_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1117940 INFO  (zkCallback-3688-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1117941 INFO  (zkCallback-3685-thread-1-processing-n:127.0.0.1:65203_solr) [n:127.0.0.1:65203_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1117941 INFO  (jetty-launcher-3657-thread-3) [n:127.0.0.1:65207_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65207_solr
   [junit4]   2> 1117943 INFO  (zkCallback-3688-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1117943 INFO  (zkCallback-3685-thread-1-processing-n:127.0.0.1:65203_solr) [n:127.0.0.1:65203_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1117943 INFO  (zkCallback-3684-thread-1-processing-n:127.0.0.1:65202_solr) [n:127.0.0.1:65202_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1117943 INFO  (zkCallback-3686-thread-1-processing-n:127.0.0.1:65207_solr) [n:127.0.0.1:65207_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1117944 INFO  (zkCallback-3687-thread-1-processing-n:127.0.0.1:65206_solr) [n:127.0.0.1:65206_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1117945 INFO  (zkCallback-3684-thread-1-processing-n:127.0.0.1:65202_solr) [n:127.0.0.1:65202_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1117945 INFO  (zkCallback-3685-thread-1-processing-n:127.0.0.1:65203_solr) [n:127.0.0.1:65203_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1117946 INFO  (zkCallback-3688-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1117946 INFO  (zkCallback-3687-thread-1-processing-n:127.0.0.1:65206_solr) [n:127.0.0.1:65206_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1117946 INFO  (zkCallback-3686-thread-1-processing-n:127.0.0.1:65207_solr) [n:127.0.0.1:65207_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1117966 INFO  (jetty-launcher-3657-thread-1) [n:127.0.0.1:65206_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65206_solr
   [junit4]   2> 1117968 INFO  (zkCallback-3688-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1117968 INFO  (zkCallback-3686-thread-1-processing-n:127.0.0.1:65207_solr) [n:127.0.0.1:65207_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1117968 INFO  (zkCallback-3684-thread-1-processing-n:127.0.0.1:65202_solr) [n:127.0.0.1:65202_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1117968 INFO  (zkCallback-3685-thread-1-processing-n:127.0.0.1:65203_solr) [n:127.0.0.1:65203_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1117968 INFO  (zkCallback-3687-thread-1-processing-n:127.0.0.1:65206_solr) [n:127.0.0.1:65206_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1117987 INFO  (jetty-launcher-3657-thread-6) [n:127.0.0.1:65204_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1117991 INFO  (jetty-launcher-3657-thread-6) [n:127.0.0.1:65204_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1117994 INFO  (jetty-launcher-3657-thread-6) [n:127.0.0.1:65204_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65204_solr
   [junit4]   2> 1117997 INFO  (zkCallback-3684-thread-1-processing-n:127.0.0.1:65202_solr) [n:127.0.0.1:65202_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1117997 INFO  (zkCallback-3686-thread-1-processing-n:127.0.0.1:65207_solr) [n:127.0.0.1:65207_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1117997 INFO  (zkCallback-3687-thread-1-processing-n:127.0.0.1:65206_solr) [n:127.0.0.1:65206_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1117997 INFO  (zkCallback-3685-thread-1-processing-n:127.0.0.1:65203_solr) [n:127.0.0.1:65203_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1117998 INFO  (zkCallback-3694-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1118000 INFO  (zkCallback-3688-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (5) -> (6)
   [junit4]   2> 1118048 INFO  (jetty-launcher-3657-thread-2) [n:127.0.0.1:65202_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node2/.
   [junit4]   2> 1118049 INFO  (jetty-launcher-3657-thread-3) [n:127.0.0.1:65207_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node3/.
   [junit4]   2> 1118051 INFO  (jetty-launcher-3657-thread-6) [n:127.0.0.1:65204_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node6/.
   [junit4]   2> 1118056 INFO  (jetty-launcher-3657-thread-5) [n:127.0.0.1:65205_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node5/.
   [junit4]   2> 1118059 INFO  (jetty-launcher-3657-thread-4) [n:127.0.0.1:65203_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node4/.
   [junit4]   2> 1118061 INFO  (jetty-launcher-3657-thread-1) [n:127.0.0.1:65206_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node1/.
   [junit4]   2> 1118074 INFO  (SUITE-ReplaceNodeTest-seed#[1F5880993BA87A28]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (6)
   [junit4]   2> 1118076 INFO  (SUITE-ReplaceNodeTest-seed#[1F5880993BA87A28]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:65200/solr ready
   [junit4]   2> 1118114 INFO  (TEST-ReplaceNodeTest.test-seed#[1F5880993BA87A28]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 1118117 INFO  (TEST-ReplaceNodeTest.test-seed#[1F5880993BA87A28]) [    ] o.a.s.c.ReplaceNodeTest total_jettys: 6
   [junit4]   2> 1118122 INFO  (qtp325148586-14985) [n:127.0.0.1:65202_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf1&maxShardsPerNode=3&name=replacenodetest_coll&action=CREATE&numShards=5&createNodeSet=127.0.0.1:65203_solr,127.0.0.1:65206_solr,127.0.0.1:65204_solr,127.0.0.1:65202_solr,127.0.0.1:65205_solr&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1118124 INFO  (OverseerThreadFactory-6649-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.CreateCollectionCmd Create collection replacenodetest_coll
   [junit4]   2> 1118233 INFO  (qtp1098862330-14970) [n:127.0.0.1:65205_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard1_replica2&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1118234 INFO  (qtp325148586-14986) [n:127.0.0.1:65202_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard1_replica1&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1118234 INFO  (qtp325148586-14986) [n:127.0.0.1:65202_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1118234 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard3_replica2&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard3&wt=javabin&version=2
   [junit4]   2> 1118236 INFO  (qtp1098862330-14970) [n:127.0.0.1:65205_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1118236 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard5_replica2&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard5&wt=javabin&version=2
   [junit4]   2> 1118236 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1118238 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard2_replica1&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard2&wt=javabin&version=2
   [junit4]   2> 1118238 INFO  (qtp2025487778-15002) [n:127.0.0.1:65203_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard4_replica2&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard4&wt=javabin&version=2
   [junit4]   2> 1118238 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1118241 INFO  (qtp944390549-14993) [n:127.0.0.1:65206_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard3_replica1&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard3&wt=javabin&version=2
   [junit4]   2> 1118241 INFO  (qtp1098862330-15015) [n:127.0.0.1:65205_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard4_replica1&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard4&wt=javabin&version=2
   [junit4]   2> 1118246 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard5_replica1&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard5&wt=javabin&version=2
   [junit4]   2> 1118246 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard2_replica2&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard2&wt=javabin&version=2
   [junit4]   2> 1118246 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1118365 INFO  (zkCallback-3684-thread-1-processing-n:127.0.0.1:65202_solr) [n:127.0.0.1:65202_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 1118365 INFO  (zkCallback-3685-thread-1-processing-n:127.0.0.1:65203_solr) [n:127.0.0.1:65203_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 1118365 INFO  (zkCallback-3687-thread-1-processing-n:127.0.0.1:65206_solr) [n:127.0.0.1:65206_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 1118365 INFO  (zkCallback-3694-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 1118365 INFO  (zkCallback-3688-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/replacenodetest_coll/state.json] for collection [replacenodetest_coll] has occurred - updating... (live nodes size: [6])
   [junit4]   2> 1119259 INFO  (qtp325148586-14986) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119278 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119292 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119292 INFO  (qtp944390549-14993) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119293 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119297 INFO  (qtp2025487778-15002) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119306 INFO  (qtp1098862330-14970) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119309 INFO  (qtp1098862330-15015) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119310 INFO  (qtp944390549-14993) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica1] o.a.s.s.IndexSchema [replacenodetest_coll_shard3_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1119310 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.s.IndexSchema [replacenodetest_coll_shard2_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1119312 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.s.IndexSchema [replacenodetest_coll_shard5_replica2] Schema name=cloud-dynamic
   [junit4]   2> 1119326 INFO  (qtp2025487778-15002) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica2] o.a.s.s.IndexSchema [replacenodetest_coll_shard4_replica2] Schema name=cloud-dynamic
   [junit4]   2> 1119333 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119333 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard5_replica2' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119333 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119333 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.c.SolrCore [[replacenodetest_coll_shard5_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node1/replacenodetest_coll_shard5_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node1/./replacenodetest_coll_shard5_replica2/data/]
   [junit4]   2> 1119335 INFO  (qtp944390549-14993) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119335 INFO  (qtp944390549-14993) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica1] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard3_replica1' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119335 INFO  (qtp944390549-14993) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119335 INFO  (qtp944390549-14993) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica1] o.a.s.c.SolrCore [[replacenodetest_coll_shard3_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node1/replacenodetest_coll_shard3_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node1/./replacenodetest_coll_shard3_replica1/data/]
   [junit4]   2> 1119335 INFO  (qtp325148586-14986) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica1] o.a.s.s.IndexSchema [replacenodetest_coll_shard1_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1119338 INFO  (qtp1098862330-15015) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica1] o.a.s.s.IndexSchema [replacenodetest_coll_shard4_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1119341 INFO  (qtp1098862330-14970) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica2] o.a.s.s.IndexSchema [replacenodetest_coll_shard1_replica2] Schema name=cloud-dynamic
   [junit4]   2> 1119341 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119357 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.s.IndexSchema [replacenodetest_coll_shard3_replica2] Schema name=cloud-dynamic
   [junit4]   2> 1119362 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 1119364 INFO  (qtp1098862330-15015) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119365 INFO  (qtp1098862330-15015) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica1] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard4_replica1' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119365 INFO  (qtp1098862330-15015) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119365 INFO  (qtp1098862330-15015) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica1] o.a.s.c.SolrCore [[replacenodetest_coll_shard4_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node5/replacenodetest_coll_shard4_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node5/./replacenodetest_coll_shard4_replica1/data/]
   [junit4]   2> 1119368 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.s.IndexSchema [replacenodetest_coll_shard2_replica2] Schema name=cloud-dynamic
   [junit4]   2> 1119368 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica1] o.a.s.s.IndexSchema [replacenodetest_coll_shard5_replica1] Schema name=cloud-dynamic
   [junit4]   2> 1119392 INFO  (qtp1098862330-14970) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119392 INFO  (qtp1098862330-14970) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard1_replica2' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119392 INFO  (qtp1098862330-14970) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119392 INFO  (qtp1098862330-14970) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica2] o.a.s.c.SolrCore [[replacenodetest_coll_shard1_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node5/replacenodetest_coll_shard1_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node5/./replacenodetest_coll_shard1_replica2/data/]
   [junit4]   2> 1119414 INFO  (qtp2025487778-15002) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119414 INFO  (qtp2025487778-15002) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica2] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard4_replica2' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119415 INFO  (qtp2025487778-15002) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119415 INFO  (qtp2025487778-15002) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica2] o.a.s.c.SolrCore [[replacenodetest_coll_shard4_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node4/replacenodetest_coll_shard4_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node4/./replacenodetest_coll_shard4_replica2/data/]
   [junit4]   2> 1119416 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119416 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard3_replica2' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119416 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119417 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.c.SolrCore [[replacenodetest_coll_shard3_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node2/replacenodetest_coll_shard3_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node2/./replacenodetest_coll_shard3_replica2/data/]
   [junit4]   2> 1119418 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119418 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard2_replica2' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119418 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119418 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.c.SolrCore [[replacenodetest_coll_shard2_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node6/replacenodetest_coll_shard2_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node6/./replacenodetest_coll_shard2_replica2/data/]
   [junit4]   2> 1119419 INFO  (qtp325148586-14986) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119419 INFO  (qtp325148586-14986) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard1_replica1' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119420 INFO  (qtp325148586-14986) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119420 INFO  (qtp325148586-14986) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard1  x:replacenodetest_coll_shard1_replica1] o.a.s.c.SolrCore [[replacenodetest_coll_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node2/replacenodetest_coll_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node2/./replacenodetest_coll_shard1_replica1/data/]
   [junit4]   2> 1119420 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119420 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard2_replica1' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119420 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119420 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.c.SolrCore [[replacenodetest_coll_shard2_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node4/replacenodetest_coll_shard2_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node4/./replacenodetest_coll_shard2_replica1/data/]
   [junit4]   2> 1119422 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1119422 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica1] o.a.s.c.CoreContainer Creating SolrCore 'replacenodetest_coll_shard5_replica1' using configuration from collection replacenodetest_coll, trusted=true
   [junit4]   2> 1119423 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1119423 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica1] o.a.s.c.SolrCore [[replacenodetest_coll_shard5_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node6/replacenodetest_coll_shard5_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ReplaceNodeTest_1F5880993BA87A28-001/tempDir-001/node6/./replacenodetest_coll_shard5_replica1/data/]
   [junit4]   2> 1119587 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.c.SolrCore [replacenodetest_coll_shard3_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@7caa05b5
   [junit4]   2> 1119588 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.c.SolrCore [replacenodetest_coll_shard5_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@121a6bd7
   [junit4]   2> 1119588 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.replacenodetest_coll.shard3.replica2
   [junit4]   2> 1119588 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.replacenodetest_coll.shard5.replica2
   [junit4]   2> 1119588 INFO  (qtp2025487778-15002) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica2] o.a.s.c.SolrCore [replacenodetest_coll_shard4_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@6ad983f4
   [junit4]   2> 1119588 INFO  (qtp2025487778-15002) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.replacenodetest_coll.shard4.replica2
   [junit4]   2> 1119589 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.c.SolrCore [replacenodetest_coll_shard2_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@2ac42356
   [junit4]   2> 1119589 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.c.SolrCore [replacenodetest_coll_shard2_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@76d6a0cf
   [junit4]   2> 1119589 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.replacenodetest_coll.shard2.replica2
   [junit4]   2> 1119590 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.replacenodetest_coll.shard2.replica1
   [junit4]   2> 1119590 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica1] o.a.s.c.SolrCore [replacenodetest_coll_shard5_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@17e93670
   [junit4]   2> 1119590 INFO  (qtp1098862330-15015) [n:127.0.0.1:65205_solr c:replacenodetest_coll s:shard4  x:replacenodetest_coll_shard4_replica1] o.a.s.c.SolrCore [replacenodetest_coll_shard4_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@30f266ac
   [junit4]   2> 1119590 INFO  (qtp1871629007-14974) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.replacenodetest_coll.shard5.replica1
   [junit4]   2> 1119608 ERROR (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'replacenodetest_coll_shard2_replica2': Unable to create core [replacenodetest_coll_shard2_replica2] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:899)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:388)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:363)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:307)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [replacenodetest_coll_shard2_replica2]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:945)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:865)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:977)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:930)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.lambda$getSpec$0(ApiBag.java:229)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:64)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:72)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:215)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:272)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1119609 INFO  (qtp1871629007-14976) [n:127.0.0.1:65204_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard2_replica2&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard2&wt=javabin&version=2} status=400 QTime=1363
   [junit4]   2> 1119609 ERROR (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'replacenodetest_coll_shard2_replica1': Unable to create core [replacenodetest_coll_shard2_replica1] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:899)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:388)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:363)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:307)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [replacenodetest_coll_shard2_replica1]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:945)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:865)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:977)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:930)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.lambda$getSpec$0(ApiBag.java:229)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:64)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:72)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:215)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:272)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1119612 INFO  (qtp2025487778-15003) [n:127.0.0.1:65203_solr c:replacenodetest_coll s:shard2  x:replacenodetest_coll_shard2_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard2_replica1&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard2&wt=javabin&version=2} status=400 QTime=1373
   [junit4]   2> 1119611 ERROR (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'replacenodetest_coll_shard5_replica2': Unable to create core [replacenodetest_coll_shard5_replica2] Caused by: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:899)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:388)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:363)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:307)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]   2> 	at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [replacenodetest_coll_shard5_replica2]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:945)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:865)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:977)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:930)
   [junit4]   2> 	... 36 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.lambda$getSpec$0(ApiBag.java:229)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:64)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:72)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:215)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:272)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
   [junit4]   2> 	... 38 more
   [junit4]   2> 
   [junit4]   2> 1119610 ERROR (OverseerThreadFactory-6649-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:65204/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:65204/solr: Error CREATEing SolrCore 'replacenodetest_coll_shard2_replica2': Unable to create core [replacenodetest_coll_shard2_replica2] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:612)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:163)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1119612 INFO  (qtp944390549-14983) [n:127.0.0.1:65206_solr c:replacenodetest_coll s:shard5  x:replacenodetest_coll_shard5_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard5_replica2&action=CREATE&numShards=5&collection=replacenodetest_coll&shard=shard5&wt=javabin&version=2} status=400 QTime=1376
   [junit4]   2> 1119612 ERROR (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'replacenodetest_coll_shard3_replica2': Unable to create core [replacenodetest_coll_shard3_replica2] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:899)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:388)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:363)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:307)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:395)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [replacenodetest_coll_shard3_replica2]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:945)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:865)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:977)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:930)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.lambda$getSpec$0(ApiBag.java:229)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:64)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:72)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:215)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:272)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:935)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1119613 ERROR (OverseerThreadFactory-6649-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:65203/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:65203/solr: Error CREATEing SolrCore 'replacenodetest_coll_shard2_replica1': Unable to create core [replacenodetest_coll_shard2_replica1] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:612)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:163)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1119616 ERROR (OverseerThreadFactory-6649-thread-1-processing-n:127.0.0.1:65205_solr) [n:127.0.0.1:65205_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:65206/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:65206/solr: Error CREATEing SolrCore 'replacenodetest_coll_shard5_replica2': Unable to create core [replacenodetest_coll_shard5_replica2] Caused by: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:612)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:163)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1119616 INFO  (qtp325148586-14987) [n:127.0.0.1:65202_solr c:replacenodetest_coll s:shard3  x:replacenodetest_coll_shard3_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=replacenodetest_coll_shard3_replica2&action=CREA

[...truncated too long message...]

tMaxTokenLenTokenizer-seed#[1F5880993BA87A28]-worker) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2403911 INFO  (coreCloseExecutor-11802-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@62c91d9e
   [junit4]   2> 2403915 INFO  (coreCloseExecutor-11802-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.util.TestMaxTokenLenTokenizer_1F5880993BA87A28-001
   [junit4]   2> NOTE: test params are: codec=Lucene62, sim=RandomSimilarity(queryNorm=true,coord=yes): {}, locale=de-GR, timezone=Asia/Vientiane
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=3,threads=1,free=135326088,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [BasicAuthIntegrationTest, MoreLikeThisHandlerTest, TestCopyFieldCollectionResource, UUIDUpdateProcessorFallbackTest, MetricUtilsTest, SolrSlf4jReporterTest, TestNumericTerms64, PropertiesRequestHandlerTest, CheckHdfsIndexTest, TestRandomRequestDistribution, TestLockTree, TestNamedUpdateProcessors, TestFaceting, TestPerFieldSimilarity, TestRemoteStreaming, TestReRankQParserPlugin, CollectionsAPIAsyncDistributedZkTest, TestJsonFacetRefinement, TestJsonRequest, DirectUpdateHandlerTest, TestSolrConfigHandlerConcurrent, TestUpdate, SharedFSAutoReplicaFailoverTest, DisMaxRequestHandlerTest, TestReplicaProperties, TestTolerantSearch, HttpSolrCallGetCoreTest, TestPhraseSuggestions, LeaderFailoverAfterPartitionTest, DeleteStatusTest, BadCopyFieldTest, TestTolerantUpdateProcessorRandomCloud, DistributedDebugComponentTest, TestLazyCores, TestCustomDocTransformer, TestMacros, TestCloudInspectUtil, TestReloadAndDeleteDocs, TestPointFields, TestDocumentBuilder, SpatialRPTFieldTypeTest, BadComponentTest, DateFieldTest, TestManagedSynonymFilterFactory, ActionThrottleTest, TestMaxScoreQueryParser, TestFastLRUCache, SaslZkACLProviderTest, TestDocBasedVersionConstraints, TestCoreContainer, InfoHandlerTest, TestRTGBase, TestSolrQueryParserDefaultOperatorResource, TestLRUCache, TestRawResponseWriter, SystemInfoHandlerTest, TestElisionMultitermQuery, BlockJoinFacetDistribTest, HdfsDirectoryTest, DocValuesNotIndexedTest, TestCustomSort, BitVectorTest, SoftAutoCommitTest, TestSchemaVersionResource, TestLeaderInitiatedRecoveryThread, ShardRoutingTest, TestDistribDocBasedVersion, DistributedSpellCheckComponentTest, SubstringBytesRefFilterTest, OverseerModifyCollectionTest, TestPayloadScoreQParserPlugin, TestSizeLimitedDistributedMap, TestJettySolrRunner, SchemaVersionSpecificBehaviorTest, DocumentAnalysisRequestHandlerTest, TestDistribIDF, TestAuthenticationFramework, TestPseudoReturnFields, TestSolrDeletionPolicy1, ChangedSchemaMergeTest, DistributedQueueTest, SolrGraphiteReporterTest, TestCloudPivotFacet, IndexBasedSpellCheckerTest, TestReplicationHandlerBackup, DeleteLastCustomShardedReplicaTest, SampleTest, VersionInfoTest, SolrRequestParserTest, TestBackupRepositoryFactory, DeleteReplicaTest, SolrMetricsIntegrationTest, TestComplexPhraseLeadingWildcard, TestHighlightDedupGrouping, SolrMetricManagerTest, ClassificationUpdateProcessorTest, TestEmbeddedSolrServerConstructors, MoveReplicaTest, TestJavabinTupleStreamParser, TestCloudSchemaless, OutputWriterTest, TestRestoreCore, PrimUtilsTest, DistributedFacetPivotSmallTest, TestSchemalessBufferedUpdates, TestHashQParserPlugin, HdfsDirectoryFactoryTest, TestSchemaResource, TestExtendedDismaxParser, TestSubQueryTransformer, QueryEqualityTest, TestLegacyFieldCache, TestStressLiveNodes, TestAnalyzeInfixSuggestions, AddBlockUpdateTest, TestSegmentSorting, JsonValidatorTest, FileUtilsTest, TestPartialUpdateDeduplication, TestReplicationHandler, TestBinaryField, CustomCollectionTest, TestMultiWordSynonyms, TestScoreJoinQPNoScore, TestStressUserVersions, ZkNodePropsTest, TestRandomDVFaceting, ClusterStateUpdateTest, TestSubQueryTransformerDistrib, TestSolrDynamicMBean, IndexSchemaRuntimeFieldTest, TestExactStatsCache, RequiredFieldsTest, TestManagedSchemaAPI, TestFieldResource, XmlUpdateRequestHandlerTest, TestReloadDeadlock, TestRequestStatusCollectionAPI, TestUniqueKeyFieldResource, CurrencyFieldXmlFileTest, ResponseLogComponentTest, HdfsThreadLeakTest, TestCSVLoader, HttpPartitionTest, FastVectorHighlighterTest, TestTrackingShardHandlerFactory, DistributedMLTComponentTest, SimpleMLTQParserTest, JSONWriterTest, TestPerFieldSimilarityClassic, TestSha256AuthenticationProvider, TestComponentsName, TestStressLucene, TestOnReconnectListenerSupport, SyncSliceTest, PingRequestHandlerTest, InfixSuggestersTest, TestConfigsApi, HdfsBasicDistributedZk2Test, SolrCoreTest, TestFieldTypeResource, TestPostingsSolrHighlighter, ConjunctionSolrSpellCheckerTest, TestApiFramework, XsltUpdateRequestHandlerTest, OpenExchangeRatesOrgProviderTest, DistribDocExpirationUpdateProcessorTest, SolrCloudExampleTest, TestSimpleTextCodec, DistributedFacetPivotLongTailTest, V2ApiIntegrationTest, LeaderInitiatedRecoveryOnCommitTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, ZkSolrClientTest, TestHashPartitioner, TermVectorComponentDistributedTest, ZkControllerTest, TestStressReorder, TestStressVersions, HardAutoCommitTest, DistributedTermsComponentTest, TestRangeQuery, SimpleFacetsTest, StatsComponentTest, SpellCheckComponentTest, TestGroupingSearch, QueryElevationComponentTest, PeerSyncTest, ConvertedLegacyTest, TestSort, TestFunctionQuery, TestIndexSearcher, HighlighterTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, SimplePostToolTest, AnalysisAfterCoreReloadTest, SuggesterFSTTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTSTTest, SuggesterTest, SpatialFilterTest, PolyFieldTest, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, DebugComponentTest, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, TestOmitPositions, RequestHandlersTest, TestValueSourceCache, TestSolrQueryParser, IndexSchemaTest, TestCollationField, TestCSVResponseWriter, TestAnalyzedSuggestions, TestPHPSerializedResponseWriter, TestBinaryResponseWriter, HighlighterConfigTest, SOLR749Test, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, CopyFieldTest, TestStressRecovery, TestSystemIdResolver, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestSolrJ, TestUtils, UUIDFieldTest, CursorPagingTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestPathTrie, AliasIntegrationTest, AssignTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CollectionStateFormat2Test, ConnectionManagerTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, OverseerStatusTest, PeerSyncReplicationTest, RemoteQueryErrorTest, ReplicationFactorTest, RestartWhileUpdatingTest, RollingRestartTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, TestCloudJSONFacetJoinDomain, TestCloudRecovery, TestClusterProperties, TestCollectionAPI, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDownShardTolerantSearch, TestExclusionRuleCollectionAccess, TestLeaderElectionWithEmptyReplica, TestLeaderElectionZkExpiry, TestLocalFSCloudBackupRestore, TestMiniSolrCloudCluster, TestRebalanceLeaders, TestTolerantUpdateProcessorCloud, VMParamsZkACLAndCredentialsProvidersTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoverLeaseTest, HdfsRecoveryZkTest, HdfsRestartWhileUpdatingTest, HdfsSyncSliceTest, HdfsTlogReplayBufferedWhileIndexingTest, HdfsUnloadDistributedZkTest, StressHdfsTest, TestClusterStateMutator, ZkStateReaderTest, CachingDirectoryFactoryTest, ConfigureRecoveryStrategyTest, CoreSorterTest, DirectoryFactoryTest, ExitableDirectoryReaderTest, QueryResultKeyTest, TestConfigOverlay, TestConfigSetImmutable, TestConfigSetProperties, TestConfigSets, TestCorePropertiesReload, TestCustomStream, TestSolrCloudSnapshots, CoreAdminCreateDiscoverTest, CoreAdminRequestStatusTest, MetricsHandlerTest, SecurityConfHandlerTest, SegmentsInfoRequestHandlerTest, StatsReloadRaceTest, TestCoreAdminApis, DistributedFacetExistsSmallTest, DistributedFacetPivotLargeTest, DistributedQueryComponentCustomSortTest, ShufflingReplicaListTransformerTest, SpatialHeatmapFacetsTest, TestExpandComponent, TestHttpShardHandlerFactory, JavabinLoaderTest, HighlighterMaxOffsetTest, TestUnifiedSolrHighlighter, TestSlowCompositeReaderWrapper, JvmMetricsTest, SolrMetricReporterTest, SolrGangliaReporterTest, TestIntervalFaceting, TestMacroExpander, JSONWriterTest, SmileWriterTest, TestBinaryResponseWriter, TestExportWriter, TestGeoJSONResponseWriter, TestSolrQueryResponse, TestChildDocTransformer, TestManagedResource, TestManagedResourceStorage, TestRestManager, TestBulkSchemaAPI, TestDynamicFieldCollectionResource, TestDynamicFieldResource, TestFieldCollectionResource, TestFieldTypeCollectionResource, TestSchemaNameResource, TestSchemaSimilarityResource, TestSolrQueryParserResource, TestManagedStopFilterFactory, TestManagedSynonymGraphFilterFactory, DocValuesMissingTest, ManagedSchemaRoundRobinCloudTest, TestCloudManagedSchema, LargeFieldTest, SortSpecParsingTest, TestCollapseQParserPlugin, TestQueryWrapperFilter, TestSimpleQParserPlugin, TestSmileRequest, TestSolr4Spatial2, TestSolrCoreParser, TestSolrFieldCacheMBean, TestXmlQParser, TestJsonFacets, ParsingFieldUpdateProcessorsTest, DistanceUnitsTest, TestMaxTokenLenTokenizer]
   [junit4] Completed [706/715 (2!)] on J0 in 0.34s, 2 tests, 1 error <<< FAILURES!

[...truncated 43496 lines...]

[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 902 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/902/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

6 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionContextKeyTest

Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO

Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at __randomizedtesting.SeedInfo.seed([CBF90CF5B8A4921B]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:270)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:264)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:188)
	at org.apache.solr.cloud.LeaderElectionContextKeyTest.setupCluster(LeaderElectionContextKeyTest.java:46)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1842)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
	at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
	at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
	at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
	... 31 more
Caused by: java.io.IOException: Too many open files
	at sun.nio.ch.IOUtil.makePipe(Native Method)
	at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
	at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
	at java.nio.channels.Selector.open(Selector.java:227)
	at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
	at sun.reflect.GeneratedConstructorAccessor219.newInstance(Unknown Source)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at java.lang.Class.newInstance(Class.java:442)
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1839)
	... 37 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionContextKeyTest

Error Message:
ObjectTracker found 3 object(s) that were not released!!! [SolrZkClient, ZkController, Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:526)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:754)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)  at org.apache.solr.cloud.ZkController.init(ZkController.java:688)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:423)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not released!!! [SolrZkClient, ZkController, Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.Overseer.start(Overseer.java:526)
	at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:754)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)
	at org.apache.solr.cloud.ZkController.init(ZkController.java:688)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:423)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([CBF90CF5B8A4921B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
	at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionContextKeyTest

Error Message:
25 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionContextKeyTest:     1) Thread[id=14038, name=jetty-launcher-2851-thread-1-EventThread, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    2) Thread[id=14011, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=14037, name=jetty-launcher-2851-thread-1-SendThread(127.0.0.1:58485), state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)    4) Thread[id=14026, name=qtp202810450-14026, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=14010, name=Thread-2846, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)    6) Thread[id=14043, name=Thread-2855, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1514)    7) Thread[id=14012, name=SessionTracker, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)    8) Thread[id=14028, name=org.eclipse.jetty.server.session.HashSessionManager@59956b5Timer, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=14019, name=qtp202810450-14019, state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=14020, name=qtp202810450-14020-acceptor-0@7ecd06dc-ServerConnector@6f8a7c38{HTTP/1.1,[http/1.1]}{127.0.0.1:58487}, state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=14040, name=OverseerStateUpdate-98066212328636419-127.0.0.1:58487_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:183)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=14032, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=14021, name=qtp202810450-14021, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=14027, name=Scheduler-1871481685, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=14024, name=qtp202810450-14024, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=14044, name=coreContainerWorkExecutor-6330-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=14023, name=qtp202810450-14023, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=14033, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=14041, name=OverseerCollectionConfigSetProcessor-98066212328636419-127.0.0.1:58487_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:360)         at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=14014, name=ProcessThread(sid:0 cport:58485):, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)   21) Thread[id=14042, name=OverseerHdfsCoreFailoverThread-98066212328636419-127.0.0.1:58487_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=14018, name=jetty-launcher-2851-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=14039, name=zkCallback-2858-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=14022, name=qtp202810450-14022, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   25) Thread[id=14025, name=qtp202810450-14025, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 25 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionContextKeyTest: 
   1) Thread[id=14038, name=jetty-launcher-2851-thread-1-EventThread, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   2) Thread[id=14011, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=14037, name=jetty-launcher-2851-thread-1-SendThread(127.0.0.1:58485), state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   4) Thread[id=14026, name=qtp202810450-14026, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=14010, name=Thread-2846, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   6) Thread[id=14043, name=Thread-2855, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1514)
   7) Thread[id=14012, name=SessionTracker, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   8) Thread[id=14028, name=org.eclipse.jetty.server.session.HashSessionManager@59956b5Timer, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=14019, name=qtp202810450-14019, state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=14020, name=qtp202810450-14020-acceptor-0@7ecd06dc-ServerConnector@6f8a7c38{HTTP/1.1,[http/1.1]}{127.0.0.1:58487}, state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=14040, name=OverseerStateUpdate-98066212328636419-127.0.0.1:58487_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:183)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=14032, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=14021, name=qtp202810450-14021, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=14027, name=Scheduler-1871481685, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=14024, name=qtp202810450-14024, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=14044, name=coreContainerWorkExecutor-6330-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=14023, name=qtp202810450-14023, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=14033, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
        at java.lang.Thread.run(Thread.java:748)
  19) Thread[id=14041, name=OverseerCollectionConfigSetProcessor-98066212328636419-127.0.0.1:58487_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:360)
        at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=14014, name=ProcessThread(sid:0 cport:58485):, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
  21) Thread[id=14042, name=OverseerHdfsCoreFailoverThread-98066212328636419-127.0.0.1:58487_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:748)
  22) Thread[id=14018, name=jetty-launcher-2851-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  23) Thread[id=14039, name=zkCallback-2858-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=14022, name=qtp202810450-14022, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  25) Thread[id=14025, name=qtp202810450-14025, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([CBF90CF5B8A4921B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionContextKeyTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=14028, name=org.eclipse.jetty.server.session.HashSessionManager@59956b5Timer, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=14037, name=jetty-launcher-2851-thread-1-SendThread(127.0.0.1:58485), state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    3) Thread[id=14019, name=qtp202810450-14019, state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.KQueueSelectorImpl.updateSelectedKeys(KQueueSelectorImpl.java:181)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=14018, name=jetty-launcher-2851-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=14027, name=Scheduler-1871481685, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=14039, name=zkCallback-2858-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=14043, name=Thread-2855, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1514)    8) Thread[id=14044, name=coreContainerWorkExecutor-6330-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=14028, name=org.eclipse.jetty.server.session.HashSessionManager@59956b5Timer, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=14037, name=jetty-launcher-2851-thread-1-SendThread(127.0.0.1:58485), state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   3) Thread[id=14019, name=qtp202810450-14019, state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]
        at sun.nio.ch.IOUtil.drain(Native Method)
        at sun.nio.ch.KQueueSelectorImpl.updateSelectedKeys(KQueueSelectorImpl.java:181)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=14018, name=jetty-launcher-2851-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=14027, name=Scheduler-1871481685, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=14039, name=zkCallback-2858-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=14043, name=Thread-2855, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1514)
   8) Thread[id=14044, name=coreContainerWorkExecutor-6330-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([CBF90CF5B8A4921B]:0)


FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:
Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>

Stack Trace:
java.lang.AssertionError: Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
	at __randomizedtesting.SeedInfo.seed([CBF90CF5B8A4921B:43AD332F1658FFE3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotSame(Assert.java:641)
	at org.junit.Assert.assertSame(Assert.java:580)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:250)
	at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ReplaceNodeTest.test

Error Message:
org/apache/solr/client/solrj/request/CollectionAdminRequest$ReplaceNode

Stack Trace:
java.lang.NoClassDefFoundError: org/apache/solr/client/solrj/request/CollectionAdminRequest$ReplaceNode
	at __randomizedtesting.SeedInfo.seed([CBF90CF5B8A4921B:43AD332F1658FFE3]:0)
	at org.apache.solr.cloud.ReplaceNodeTest.test(ReplaceNodeTest.java:67)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.apache.solr.client.solrj.request.CollectionAdminRequest$ReplaceNode
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 40 more
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-solrj/classes/java/org/apache/solr/client/solrj/request/CollectionAdminRequest$ReplaceNode.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 46 more




Build Log:
[...truncated 12040 lines...]
   [junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_CBF90CF5B8A4921B-001/init-core-data-001
   [junit4]   2> 1340771 WARN  (SUITE-MultiThreadedOCPTest-seed#[CBF90CF5B8A4921B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 1340771 INFO  (SUITE-MultiThreadedOCPTest-seed#[CBF90CF5B8A4921B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 1340774 INFO  (SUITE-MultiThreadedOCPTest-seed#[CBF90CF5B8A4921B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1340775 INFO  (SUITE-MultiThreadedOCPTest-seed#[CBF90CF5B8A4921B]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /sl/
   [junit4]   2> 1340776 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1340776 INFO  (Thread-2487) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1340776 INFO  (Thread-2487) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1340779 ERROR (Thread-2487) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1340883 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.ZkTestServer start zk server on port:57903
   [junit4]   2> 1340911 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1340914 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1340916 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1340918 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1340920 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1340922 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1340924 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1340926 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1340928 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1340930 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1340932 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1341273 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_CBF90CF5B8A4921B-001/control-001/cores/collection1
   [junit4]   2> 1341276 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1341279 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@cc9a03d{/sl,null,AVAILABLE}
   [junit4]   2> 1341279 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5ad777f8{HTTP/1.1,[http/1.1]}{127.0.0.1:57908}
   [junit4]   2> 1341280 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.e.j.s.Server Started @1345002ms
   [junit4]   2> 1341280 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_CBF90CF5B8A4921B-001/tempDir-001/control/data, hostContext=/sl, hostPort=57908, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_CBF90CF5B8A4921B-001/control-001/cores}
   [junit4]   2> 1341280 ERROR (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1341281 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.7.0
   [junit4]   2> 1341281 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1341281 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1341281 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-06-02T02:44:40.519Z
   [junit4]   2> 1341284 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1341285 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_CBF90CF5B8A4921B-001/control-001/solr.xml
   [junit4]   2> 1341293 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1341294 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57903/solr
   [junit4]   2> 1341324 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [n:127.0.0.1:57908_sl    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1341325 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [n:127.0.0.1:57908_sl    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57908_sl
   [junit4]   2> 1341327 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [n:127.0.0.1:57908_sl    ] o.a.s.c.Overseer Overseer (id=98066201314328580-127.0.0.1:57908_sl-n_0000000000) starting
   [junit4]   2> 1341337 INFO  (TEST-MultiThreadedOCPTest.test-seed#[CBF90CF5B8A4921B]) [n:127.0.0.1:57908_sl    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57908_sl
   [junit4]   2> 1341339 

[...truncated too long message...]

   at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   25) Thread[id=14025, name=qtp202810450-14025, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CBF90CF5B8A4921B]:0)Throwable #4: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=14028, name=org.eclipse.jetty.server.session.HashSessionManager@59956b5Timer, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=14037, name=jetty-launcher-2851-thread-1-SendThread(127.0.0.1:58485), state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >    3) Thread[id=14019, name=qtp202810450-14019, state=RUNNABLE, group=TGRP-LeaderElectionContextKeyTest]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.updateSelectedKeys(KQueueSelectorImpl.java:181)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=14018, name=jetty-launcher-2851-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=14027, name=Scheduler-1871481685, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=14039, name=zkCallback-2858-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=14043, name=Thread-2855, state=WAITING, group=TGRP-LeaderElectionContextKeyTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:502)
   [junit4]    >         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1514)
   [junit4]    >    8) Thread[id=14044, name=coreContainerWorkExecutor-6330-thread-1, state=TIMED_WAITING, group=TGRP-LeaderElectionContextKeyTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CBF90CF5B8A4921B]:0)
   [junit4] Completed [356/715 (3!)] on J1 in 44.85s, 0 tests, 1 failure, 3 errors <<< FAILURES!

[...truncated 44623 lines...]

[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 901 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/901/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

6 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI

Error Message:
Could not find collection : implicitcoll

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : implicitcoll
	at __randomizedtesting.SeedInfo.seed([D22AB0B0AA8D3221:B8CB3EDB97178459]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:194)
	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:245)
	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:68)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter

Error Message:
Collection not found: routeFieldColl

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: routeFieldColl
	at __randomizedtesting.SeedInfo.seed([D22AB0B0AA8D3221:7A1C2E6D35ECD97B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1401)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1094)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1073)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter(CustomCollectionTest.java:166)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DocValuesNotIndexedTest

Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO

Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at __randomizedtesting.SeedInfo.seed([D22AB0B0AA8D3221]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:270)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:264)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:188)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.createCluster(DocValuesNotIndexedTest.java:79)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1842)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
	at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
	at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
	at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
	... 31 more
Caused by: java.io.IOException: Too many open files
	at sun.nio.ch.IOUtil.makePipe(Native Method)
	at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
	at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
	at java.nio.channels.Selector.open(Selector.java:227)
	at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
	at sun.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at java.lang.Class.newInstance(Class.java:442)
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1839)
	... 37 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DocValuesNotIndexedTest

Error Message:
ObjectTracker found 5 object(s) that were not released!!! [SolrZkClient, ZkController, ZkController, Overseer, SolrZkClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:526)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:754)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)  at org.apache.solr.cloud.ZkController.init(ZkController.java:688)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:423)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [SolrZkClient, ZkController, ZkController, Overseer, SolrZkClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.Overseer.start(Overseer.java:526)
	at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:754)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)
	at org.apache.solr.cloud.ZkController.init(ZkController.java:688)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:423)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:486)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:246)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:175)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([D22AB0B0AA8D3221]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
	at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DocValuesNotIndexedTest

Error Message:
42 threads leaked from SUITE scope at org.apache.solr.cloud.DocValuesNotIndexedTest:     1) Thread[id=19112, name=jetty-launcher-3639-thread-2-EventThread, state=WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    2) Thread[id=19086, name=qtp1829883756-19086, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=19076, name=qtp1829883756-19076, state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=19119, name=Thread-4286, state=WAITING, group=TGRP-DocValuesNotIndexedTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1514)    5) Thread[id=19101, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=19113, name=jetty-launcher-3639-thread-1-EventThread, state=WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    7) Thread[id=19064, name=Thread-4269, state=WAITING, group=TGRP-DocValuesNotIndexedTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)    8) Thread[id=19093, name=org.eclipse.jetty.server.session.HashSessionManager@455b555bTimer, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=19078, name=qtp1829883756-19078-acceptor-0@fb9b81c-ServerConnector@3235502f{HTTP/1.1,[http/1.1]}{127.0.0.1:51087}, state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=19079, name=qtp1829883756-19079, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=19114, name=zkCallback-3651-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=19100, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=19073, name=jetty-launcher-3639-thread-2, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=19102, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=19080, name=qtp407587726-19080, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=19083, name=qtp1829883756-19083, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=19110, name=jetty-launcher-3639-thread-2-SendThread(127.0.0.1:51085), state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)   18) Thread[id=19077, name=qtp407587726-19077, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=19091, name=Scheduler-950121319, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=19111, name=jetty-launcher-3639-thread-1-SendThread(127.0.0.1:51085), state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)   21) Thread[id=19117, name=OverseerCollectionConfigSetProcessor-98064108011520005-127.0.0.1:51087_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:360)         at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=19066, name=SessionTracker, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   23) Thread[id=19103, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=19072, name=jetty-launcher-3639-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   25) Thread[id=19085, name=qtp407587726-19085, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   26) Thread[id=19087, name=qtp1829883756-19087, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   27) Thread[id=19088, name=qtp407587726-19088, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   28) Thread[id=19074, name=qtp407587726-19074, state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   29) Thread[id=19084, name=qtp407587726-19084, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   30) Thread[id=19120, name=coreContainerWorkExecutor-9521-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   31) Thread[id=19065, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)   32) Thread[id=19118, name=OverseerHdfsCoreFailoverThread-98064108011520005-127.0.0.1:51087_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)         at java.lang.Thread.run(Thread.java:748)   33) Thread[id=19068, name=ProcessThread(sid:0 cport:51085):, state=WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)   34) Thread[id=19089, name=qtp1829883756-19089, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   35) Thread[id=19090, name=Scheduler-1038132740, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   36) Thread[id=19121, name=coreContainerWorkExecutor-9519-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   37) Thread[id=19075, name=qtp407587726-19075-acceptor-0@6e17290a-ServerConnector@64e05ca2{HTTP/1.1,[http/1.1]}{127.0.0.1:51088}, state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   38) Thread[id=19081, name=qtp407587726-19081, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   39) Thread[id=19116, name=OverseerStateUpdate-98064108011520005-127.0.0.1:51087_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:183)         at java.lang.Thread.run(Thread.java:748)   40) Thread[id=19082, name=qtp1829883756-19082, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   41) Thread[id=19092, name=org.eclipse.jetty.server.session.HashSessionManager@2fe0e90dTimer, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   42) Thread[id=19115, name=zkCallback-3650-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 42 threads leaked from SUITE scope at org.apache.solr.cloud.DocValuesNotIndexedTest: 
   1) Thread[id=19112, name=jetty-launcher-3639-thread-2-EventThread, state=WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   2) Thread[id=19086, name=qtp1829883756-19086, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=19076, name=qtp1829883756-19076, state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=19119, name=Thread-4286, state=WAITING, group=TGRP-DocValuesNotIndexedTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1514)
   5) Thread[id=19101, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=19113, name=jetty-launcher-3639-thread-1-EventThread, state=WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   7) Thread[id=19064, name=Thread-4269, state=WAITING, group=TGRP-DocValuesNotIndexedTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   8) Thread[id=19093, name=org.eclipse.jetty.server.session.HashSessionManager@455b555bTimer, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=19078, name=qtp1829883756-19078-acceptor-0@fb9b81c-ServerConnector@3235502f{HTTP/1.1,[http/1.1]}{127.0.0.1:51087}, state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=19079, name=qtp1829883756-19079, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=19114, name=zkCallback-3651-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=19100, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=19073, name=jetty-launcher-3639-thread-2, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  14) Thread[id=19102, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=19080, name=qtp407587726-19080, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=19083, name=qtp1829883756-19083, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=19110, name=jetty-launcher-3639-thread-2-SendThread(127.0.0.1:51085), state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
  18) Thread[id=19077, name=qtp407587726-19077, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  19) Thread[id=19091, name=Scheduler-950121319, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  20) Thread[id=19111, name=jetty-launcher-3639-thread-1-SendThread(127.0.0.1:51085), state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
  21) Thread[id=19117, name=OverseerCollectionConfigSetProcessor-98064108011520005-127.0.0.1:51087_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:360)
        at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)
        at java.lang.Thread.run(Thread.java:748)
  22) Thread[id=19066, name=SessionTracker, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
  23) Thread[id=19103, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
        at java.lang.Thread.run(Thread.java:748)
  24) Thread[id=19072, name=jetty-launcher-3639-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  25) Thread[id=19085, name=qtp407587726-19085, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  26) Thread[id=19087, name=qtp1829883756-19087, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  27) Thread[id=19088, name=qtp407587726-19088, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  28) Thread[id=19074, name=qtp407587726-19074, state=RUNNABLE, group=TGRP-DocValuesNotIndexedTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        a

[...truncated too long message...]

    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=19093, name=org.eclipse.jetty.server.session.HashSessionManager@455b555bTimer, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    8) Thread[id=19114, name=zkCallback-3651-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=19090, name=Scheduler-1038132740, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=19121, name=coreContainerWorkExecutor-9519-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   11) Thread[id=19073, name=jetty-launcher-3639-thread-2, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   12) Thread[id=19110, name=jetty-launcher-3639-thread-2-SendThread(127.0.0.1:51085), state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >   13) Thread[id=19091, name=Scheduler-950121319, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   14) Thread[id=19092, name=org.eclipse.jetty.server.session.HashSessionManager@2fe0e90dTimer, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   15) Thread[id=19115, name=zkCallback-3650-thread-1, state=TIMED_WAITING, group=TGRP-DocValuesNotIndexedTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D22AB0B0AA8D3221]:0)
   [junit4] Completed [504/715 (2!)] on J1 in 44.81s, 0 tests, 1 failure, 3 errors <<< FAILURES!

[...truncated 44146 lines...]

[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 900 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/900/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

5 tests failed.
FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:
Task 3001 did not complete, final state: RUNNING expected same:<COMPLETED> was not:<RUNNING>

Stack Trace:
java.lang.AssertionError: Task 3001 did not complete, final state: RUNNING expected same:<COMPLETED> was not:<RUNNING>
	at __randomizedtesting.SeedInfo.seed([EC1C844FF8F301B:8695F79E51735DE3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotSame(Assert.java:641)
	at org.junit.Assert.assertSame(Assert.java:580)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:250)
	at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestShortCircuitedRequests.test

Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO

Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at __randomizedtesting.SeedInfo.seed([EC1C844FF8F301B:8695F79E51735DE3]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:107)
	at org.apache.solr.common.cloud.ZkStateReader.<init>(ZkStateReader.java:226)
	at org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider.connect(ZkClientClusterStateProvider.java:131)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.connect(CloudSolrClient.java:631)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.initCloud(AbstractFullDistribZkTestBase.java:267)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:329)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:990)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1842)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
	at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
	at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
	at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
	... 40 more
Caused by: java.io.IOException: Too many open files
	at sun.nio.ch.IOUtil.makePipe(Native Method)
	at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
	at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
	at java.nio.channels.Selector.open(Selector.java:227)
	at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
	at sun.reflect.GeneratedConstructorAccessor98.newInstance(Unknown Source)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at java.lang.Class.newInstance(Class.java:442)
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1839)
	... 46 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestShortCircuitedRequests

Error Message:
12 threads leaked from SUITE scope at org.apache.solr.cloud.TestShortCircuitedRequests:     1) Thread[id=19965, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=19971, name=TEST-TestShortCircuitedRequests.test-seed#[EC1C844FF8F301B]-EventThread, state=WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)    3) Thread[id=19970, name=TEST-TestShortCircuitedRequests.test-seed#[EC1C844FF8F301B]-SendThread(127.0.0.1:62909), state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    4) Thread[id=19966, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=19972, name=zkCallback-3749-thread-1, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=19975, name=OverseerHdfsCoreFailoverThread-98048830100799492-127.0.0.1:62912_-n_0000000000, state=WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)         at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1110)         at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1096)         at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:638)         at org.apache.solr.common.cloud.ClusterState.getCollectionsMap(ClusterState.java:250)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:166)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=19978, name=zkCallback-3749-thread-4, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=19974, name=OverseerCollectionConfigSetProcessor-98048830100799492-127.0.0.1:62912_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=19976, name=zkCallback-3749-thread-2, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=19977, name=zkCallback-3749-thread-3, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=19979, name=zkCallback-3749-thread-5, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=19973, name=OverseerStateUpdate-98048830100799492-127.0.0.1:62912_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:183)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 12 threads leaked from SUITE scope at org.apache.solr.cloud.TestShortCircuitedRequests: 
   1) Thread[id=19965, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=19971, name=TEST-TestShortCircuitedRequests.test-seed#[EC1C844FF8F301B]-EventThread, state=WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
   3) Thread[id=19970, name=TEST-TestShortCircuitedRequests.test-seed#[EC1C844FF8F301B]-SendThread(127.0.0.1:62909), state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   4) Thread[id=19966, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=19972, name=zkCallback-3749-thread-1, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=19975, name=OverseerHdfsCoreFailoverThread-98048830100799492-127.0.0.1:62912_-n_0000000000, state=WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1406)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1210)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:357)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1110)
        at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1096)
        at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:638)
        at org.apache.solr.common.cloud.ClusterState.getCollectionsMap(ClusterState.java:250)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:166)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:131)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=19978, name=zkCallback-3749-thread-4, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=19974, name=OverseerCollectionConfigSetProcessor-98048830100799492-127.0.0.1:62912_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=19976, name=zkCallback-3749-thread-2, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=19977, name=zkCallback-3749-thread-3, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=19979, name=zkCallback-3749-thread-5, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=19973, name=OverseerStateUpdate-98048830100799492-127.0.0.1:62912_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:183)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([EC1C844FF8F301B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestShortCircuitedRequests

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=19976, name=zkCallback-3749-thread-2, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=19977, name=zkCallback-3749-thread-3, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=19970, name=TEST-TestShortCircuitedRequests.test-seed#[EC1C844FF8F301B]-SendThread(127.0.0.1:62909), state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    4) Thread[id=19972, name=zkCallback-3749-thread-1, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=19979, name=zkCallback-3749-thread-5, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=19978, name=zkCallback-3749-thread-4, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19976, name=zkCallback-3749-thread-2, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=19977, name=zkCallback-3749-thread-3, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=19970, name=TEST-TestShortCircuitedRequests.test-seed#[EC1C844FF8F301B]-SendThread(127.0.0.1:62909), state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   4) Thread[id=19972, name=zkCallback-3749-thread-1, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=19979, name=zkCallback-3749-thread-5, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=19978, name=zkCallback-3749-thread-4, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([EC1C844FF8F301B]:0)


FAILED:  org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([EC1C844FF8F301B:6413F72BA76CE0D4]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:895)
	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">13</int></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=letter0:lett&wt=xml
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
	... 40 more




Build Log:
[...truncated 11193 lines...]
   [junit4] Suite: org.apache.solr.util.TestMaxTokenLenTokenizer
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.util.TestMaxTokenLenTokenizer_EC1C844FF8F301B-001/init-core-data-001
   [junit4]   2> 295674 WARN  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 295674 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 295678 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 295679 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 295679 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 295730 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 295749 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 295762 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 295766 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 295814 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@478ce589
   [junit4]   2> 295817 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@478ce589
   [junit4]   2> 295817 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@478ce589
   [junit4]   2> 295818 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 295864 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
   [junit4]   2> 295890 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 295897 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 295897 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 295897 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@478ce589
   [junit4]   2> 295898 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 295898 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.util.TestMaxTokenLenTokenizer_EC1C844FF8F301B-001/init-core-data-001/]
   [junit4]   2> 295898 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=75.30859375, floorSegmentMB=0.57421875, forceMergeDeletesPctAllowed=25.847654569285076, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 295952 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 295952 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 295953 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=21, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 295953 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6899713c[collection1] main]
   [junit4]   2> 295953 WARN  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 295954 INFO  (coreLoadExecutor-1873-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 295954 INFO  (searcherExecutor-1874-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6899713c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 295955 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 295956 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSingleFieldSameAnalyzers
   [junit4]   2> 295956 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 295956 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:*} 0 0
   [junit4]   2> 295957 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 295957 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 295961 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@7f020223[collection1] main]
   [junit4]   2> 295961 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 295961 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7f020223[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1)))}
   [junit4]   2> 295961 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[1],commit=} 0 4
   [junit4]   2> 295962 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 295962 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 295963 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@4a629292[collection1] main]
   [junit4]   2> 295963 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 295963 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4a629292[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1) Uninverting(_1(6.7.0):c1)))}
   [junit4]   2> 295963 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[2],commit=} 0 2
   [junit4]   2> 295964 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 295964 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 295966 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@7bf78dca[collection1] main]
   [junit4]   2> 295967 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 295967 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7bf78dca[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1) Uninverting(_1(6.7.0):c1) Uninverting(_2(6.7.0):c1)))}
   [junit4]   2> 295967 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[3],commit=} 0 3
   [junit4]   2> 295967 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 295967 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 295969 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@15e7d5c1[collection1] main]
   [junit4]   2> 295969 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 295969 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@15e7d5c1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1) Uninverting(_1(6.7.0):c1) Uninverting(_2(6.7.0):c1) Uninverting(_3(6.7.0):c1)))}
   [junit4]   2> 295969 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[4],commit=} 0 2
   [junit4]   2> 295970 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 295970 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 295971 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@63e87283[collection1] main]
   [junit4]   2> 295971 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 295972 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@63e87283[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.7.0):c1) Uninverting(_1(6.7.0):c1) Uninverting(_2(6.7.0):c1) Uninverting(_3(6.7.0):c1) Uninverting(_4(6.7.0):c1)))}
   [junit4]   2> 295972 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[5],commit=} 0 2
   [junit4]   2> 295972 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 295972 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 295972 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 295972 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 0
   [junit4]   2> 295976 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=*:*&wt=xml} hits=5 status=0 QTime=0
   [junit4]   2> 295977 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter0:l&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 295977 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter0:let&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 295991 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter0:lett&wt=xml} hits=0 status=0 QTime=13
   [junit4]   2> 295991 ERROR (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">13</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=letter0:lett&wt=xml
   [junit4]   2> 295992 ERROR (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=letter0:lett&wt=xml:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">13</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=letter0:lett&wt=xml
   [junit4]   2> 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
   [junit4]   2> 	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 295992 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSingleFieldSameAnalyzers
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestMaxTokenLenTokenizer -Dtests.method=testSingleFieldSameAnalyzers -Dtests.seed=EC1C844FF8F301B -Dtests.slow=true -Dtests.locale=fr-BE -Dtests.timezone=Africa/Johannesburg -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.04s J1 | TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during query
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EC1C844FF8F301B:6413F72BA76CE0D4]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:895)
   [junit4]    > 	at org.apache.solr.util.TestMaxTokenLenTokenizer.testSingleFieldSameAnalyzers(TestMaxTokenLenTokenizer.java:104)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]    > 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader"><int name="status">0</int><int name="QTime">13</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]    > </response>
   [junit4]    > 	request was:q=letter0:lett&wt=xml
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:888)
   [junit4]    > 	... 40 more
   [junit4]   2> 295995 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSingleFieldDiffAnalyzers
   [junit4]   2> 295995 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 295995 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={_version_=-9223372036854775807&update.distrib=FROMLEADER&versions=true&wt=json&indent=true}{deleteByQuery=*:*} 0 0
   [junit4]   2> 295996 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 295996 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 295998 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@7e8b0318[collection1] main]
   [junit4]   2> 295998 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 295998 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7e8b0318[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1)))}
   [junit4]   2> 295998 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[1],commit=} 0 2
   [junit4]   2> 295999 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 295999 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 296000 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@23b9044d[collection1] main]
   [junit4]   2> 296001 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 296001 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@23b9044d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1) Uninverting(_6(6.7.0):c1)))}
   [junit4]   2> 296001 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[2],commit=} 0 2
   [junit4]   2> 296001 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 296001 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 296003 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@259ea61c[collection1] main]
   [junit4]   2> 296003 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 296003 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@259ea61c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1) Uninverting(_6(6.7.0):c1) Uninverting(_7(6.7.0):c1)))}
   [junit4]   2> 296003 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[3],commit=} 0 2
   [junit4]   2> 296004 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 296004 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 296005 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@25325c4e[collection1] main]
   [junit4]   2> 296005 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 296006 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@25325c4e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1) Uninverting(_6(6.7.0):c1) Uninverting(_7(6.7.0):c1) Uninverting(_8(6.7.0):c1)))}
   [junit4]   2> 296006 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[4],commit=} 0 2
   [junit4]   2> 296007 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 296007 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@ff7831e
   [junit4]   2> 296009 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@323bd3c[collection1] main]
   [junit4]   2> 296009 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 296009 INFO  (searcherExecutor-1874-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@323bd3c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.7.0):c1) Uninverting(_6(6.7.0):c1) Uninverting(_7(6.7.0):c1) Uninverting(_8(6.7.0):c1) Uninverting(_9(6.7.0):c1)))}
   [junit4]   2> 296009 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={wt=json&indent=true}{add=[5],commit=} 0 3
   [junit4]   2> 296009 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 296009 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 296010 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 296010 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 0
   [junit4]   2> 296010 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=*:*&wt=xml} hits=5 status=0 QTime=0
   [junit4]   2> 296011 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter:let&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 296011 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=letter:lett&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 296011 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:low&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 296012 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:l&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 296012 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:lo&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 296012 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=lowerCase:lower&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 296013 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:whi&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 296013 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:teS&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 296013 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:in&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 296013 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=whiteSpace:white&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 296014 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:uni&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 296014 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:cod&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 296014 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:e&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 296014 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=unicodeWhiteSpace:unico&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 296015 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=keyword:keyword&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 296015 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=keyword:key&wt=xml} hits=0 status=0 QTime=0
   [junit4]   2> 296015 INFO  (TEST-TestMaxTokenLenTokenizer.testSingleFieldDiffAnalyzers-seed#[EC1C844FF8F301B]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSingleFieldDiffAnalyzers
   [junit4]   2> 296015 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 296015 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=18711590
   [junit4]   2> 296015 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 296016 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.jvm
   [junit4]   2> 296016 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.jetty
   [junit4]   2> 296017 INFO  (SUITE-TestMaxTokenLenTokenizer-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 296017 INFO  (coreCloseExecutor-1878-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@35c51c67
   [junit4]   2> 296018 INFO  (coreCloseExecutor-1878-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.util.TestMaxTokenLenTokenizer_EC1C844FF8F301B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {lowerCase0=PostingsFormat(name=LuceneFixedGap), whiteSpace=FST50, letter=PostingsFormat(name=Memory doPackFST= false), lowerCase=FST50, unicodeWhiteSpace=PostingsFormat(name=Memory doPackFST= false), letter0=Lucene50(blocksize=128), unicodeWhiteSpace0=Lucene50(blocksize=128), keyword0=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=LuceneFixedGap), keyword=FST50, whiteSpace0=PostingsFormat(name=LuceneFixedGap)}, docValues:{}, maxPointsInLeafNode=411, maxMBSortInHeap=6.357377398541461, sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=fr-BE, timezone=Africa/Johannesburg
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=3,threads=1,free=100529448,total=330301440
   [junit4]   2> NOTE: All tests run in this JVM: [SparseHLLTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestDeleteCollectionOnDownNodes, TestBulkSchemaConcurrent, TestCollectionAPIs, QueryParsingTest, TestDynamicFieldCollectionResource, CleanupOldIndexTest, TestIndexingPerformance, TestInitQParser, TestCollapseQParserPlugin, TestBadConfig, TestSolr4Spatial, EchoParamsTest, TestClusterProperties, ResourceLoaderTest, SpellingQueryConverterTest, TestDistributedMissingSort, BufferStoreTest, TestQueryUtils, OpenCloseCoreStressTest, TestSerializedLuceneMatchVersion, OutOfBoxZkACLAndCredentialsProvidersTest, TestBlendedInfixSuggestions, DirectUpdateHandlerOptimizeTest, CoreMergeIndexesAdminHandlerTest, SpellCheckCollatorTest, TestInfoStreamLogging, TestCloudJSONFacetJoinDomain, SpatialHeatmapFacetsTest, SolrXmlInZkTest, TestGraphTermsQParserPlugin, TestConfigSetProperties, JsonLoaderTest, TestConfig, ShowFileRequestHandlerTest, BasicZkTest, BasicAuthStandaloneTest, TestGraphMLResponseWriter, UpdateParamsTest, TestExactSharedStatsCache, RAMDirectoryFactoryTest, CdcrBootstrapTest, TestDynamicLoading, TestMacroExpander, TestClusterStateMutator, TestHdfsBackupRestoreCore, LeaderFailureAfterFreshStartTest, TestFieldTypeCollectionResource, TestManagedSchemaThreadSafety, TestFreeTextSuggestions, TestSolrCoreParser, TestRequestForwarding, TestMaxTokenLenTokenizer]
   [junit4] Completed [86/715 (1!)] on J1 in 0.36s, 2 tests, 1 error <<< FAILURES!

[...truncated 1502 lines...]
   [junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_EC1C844FF8F301B-001/init-core-data-001
   [junit4]   2> 1721575 WARN  (SUITE-MultiThreadedOCPTest-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
   [junit4]   2> 1721575 INFO  (SUITE-MultiThreadedOCPTest-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields
   [junit4]   2> 1721581 INFO  (SUITE-MultiThreadedOCPTest-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1721581 INFO  (SUITE-MultiThreadedOCPTest-seed#[EC1C844FF8F301B]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /vfqx/fi
   [junit4]   2> 1721583 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1721584 INFO  (Thread-3467) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1721584 INFO  (Thread-3467) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1721589 ERROR (Thread-3467) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1721694 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.ZkTestServer start zk server on port:62058
   [junit4]   2> 1721722 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1721728 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1721733 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1721737 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1721742 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1721746 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1721750 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1721755 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1721759 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1721763 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1721768 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1721999 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_EC1C844FF8F301B-001/control-001/cores/collection1
   [junit4]   2> 1722000 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1722002 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@29282040{/vfqx/fi,null,AVAILABLE}
   [junit4]   2> 1722003 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@55f5ebe3{HTTP/1.1,[http/1.1]}{127.0.0.1:62061}
   [junit4]   2> 1722003 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.e.j.s.Server Started @1725619ms
   [junit4]   2> 1722003 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_EC1C844FF8F301B-001/tempDir-001/control/data, hostContext=/vfqx/fi, hostPort=62061, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.MultiThreadedOCPTest_EC1C844FF8F301B-001/control-001/cores}
   [junit4]   2> 1722003 ERROR (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1722003 INFO  (TEST-MultiThreadedOCPTest.test-seed#[EC1C844FF8F301B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Wel

[...truncated too long message...]


   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=19976, name=zkCallback-3749-thread-2, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=19977, name=zkCallback-3749-thread-3, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   11) Thread[id=19979, name=zkCallback-3749-thread-5, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   12) Thread[id=19973, name=OverseerStateUpdate-98048830100799492-127.0.0.1:62912_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
   [junit4]    >         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
   [junit4]    >         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:183)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EC1C844FF8F301B]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=19976, name=zkCallback-3749-thread-2, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=19977, name=zkCallback-3749-thread-3, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=19970, name=TEST-TestShortCircuitedRequests.test-seed#[EC1C844FF8F301B]-SendThread(127.0.0.1:62909), state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >    4) Thread[id=19972, name=zkCallback-3749-thread-1, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=19979, name=zkCallback-3749-thread-5, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=19978, name=zkCallback-3749-thread-4, state=TIMED_WAITING, group=TGRP-TestShortCircuitedRequests]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EC1C844FF8F301B]:0)
   [junit4] Completed [544/715 (3!)] on J0 in 23.90s, 1 test, 3 errors <<< FAILURES!

[...truncated 44026 lines...]