You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/02/10 00:05:20 UTC

[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 687 - Failure!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/687/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

2 tests failed.
FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
	at __randomizedtesting.SeedInfo.seed([46FAF5795D7DA6FC:CDDD26A81C7B0D78]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:187)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
	at org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:437)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.admin.MBeansHandlerTest.testDiff

Error Message:
expected:<Was: [1, Now: 2], Delta: 1> but was:<Was: [4, Now: 5], Delta: 1>

Stack Trace:
org.junit.ComparisonFailure: expected:<Was: [1, Now: 2], Delta: 1> but was:<Was: [4, Now: 5], Delta: 1>
	at __randomizedtesting.SeedInfo.seed([46FAF5795D7DA6FC:83EC31E24DCB9E9C]:0)
	at org.junit.Assert.assertEquals(Assert.java:125)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.admin.MBeansHandlerTest.testDiff(MBeansHandlerTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12135 lines...]
   [junit4] Suite: org.apache.solr.handler.admin.MBeansHandlerTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.admin.MBeansHandlerTest_46FAF5795D7DA6FC-001/init-core-data-001
   [junit4]   2> 2346362 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2346362 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 2346363 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2346424 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2346442 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 2346538 WARN  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.s.IndexSchema [null] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2346542 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2346552 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 2346602 WARN  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2346604 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@22faa279
   [junit4]   2> 2346604 WARN  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2346607 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@22faa279
   [junit4]   2> 2346607 WARN  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2346607 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@22faa279
   [junit4]   2> 2346609 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2346676 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2346695 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2346766 WARN  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2346769 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2346784 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1
   [junit4]   2> 2346784 WARN  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2346784 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@22faa279
   [junit4]   2> 2346785 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.admin.MBeansHandlerTest_46FAF5795D7DA6FC-001/init-core-data-001/]
   [junit4]   2> 2346785 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@22faa279
   [junit4]   2> 2346787 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 2346886 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2346886 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2346887 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2346887 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2346888 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=81.7421875, floorSegmentMB=0.87890625, forceMergeDeletesPctAllowed=26.434065076382023, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37115584536871093
   [junit4]   2> 2346890 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e5dc1d0[collection1] main]
   [junit4]   2> 2346890 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 2346897 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 2346933 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2346933 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 2346933 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 2346934 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 2346934 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 2346934 INFO  (coreLoadExecutor-8249-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558899174489980928
   [junit4]   2> 2346935 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 2346935 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 2346936 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 2346936 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 2346937 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 2346938 INFO  (searcherExecutor-8250-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6e5dc1d0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2346939 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 2346942 INFO  (TEST-MBeansHandlerTest.testXMLDiffWithExternalEntity-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testXMLDiffWithExternalEntity
   [junit4]   2> 2346946 INFO  (TEST-MBeansHandlerTest.testXMLDiffWithExternalEntity-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testXMLDiffWithExternalEntity
   [junit4]   2> 2346948 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBackCompatNames
   [junit4]   2> 2346948 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&wt=xml} status=0 QTime=0
   [junit4]   2> 2346956 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&cat=QUERYHANDLER&wt=xml} status=0 QTime=0
   [junit4]   2> 2346957 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&cat=UPDATE&wt=xml} status=0 QTime=0
   [junit4]   2> 2346957 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBackCompatNames
   [junit4]   2> 2346959 INFO  (TEST-MBeansHandlerTest.testDiff-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDiff
   [junit4]   2> 2346960 INFO  (TEST-MBeansHandlerTest.testDiff-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&stats=true&wt=xml} status=0 QTime=0
   [junit4]   2> 2346987 INFO  (TEST-MBeansHandlerTest.testDiff-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&stats=true&diff=true&wt=xml} status=0 QTime=26
   [junit4]   2> 2346989 INFO  (TEST-MBeansHandlerTest.testDiff-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDiff
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=MBeansHandlerTest -Dtests.method=testDiff -Dtests.seed=46FAF5795D7DA6FC -Dtests.slow=true -Dtests.locale=es-ES -Dtests.timezone=America/Lower_Princes -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.04s J1 | MBeansHandlerTest.testDiff <<<
   [junit4]    > Throwable #1: org.junit.ComparisonFailure: expected:<Was: [1, Now: 2], Delta: 1> but was:<Was: [4, Now: 5], Delta: 1>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([46FAF5795D7DA6FC:83EC31E24DCB9E9C]:0)
   [junit4]    > 	at org.apache.solr.handler.admin.MBeansHandlerTest.testDiff(MBeansHandlerTest.java:63)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2346998 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 2346999 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2028281771
   [junit4]   2> 2347002 INFO  (coreCloseExecutor-8255-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@2a75eec8
   [junit4]   2> 2347132 INFO  (coreCloseExecutor-8255-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 2347135 INFO  (SUITE-MBeansHandlerTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.handler.admin.MBeansHandlerTest_46FAF5795D7DA6FC-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=es-ES, timezone=America/Lower_Princes
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=364526800,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestRestoreCore, TestManagedSchema, TestDownShardTolerantSearch, TestUninvertingReader, HdfsRecoverLeaseTest, FullSolrCloudDistribCmdsTest, TestStressLiveNodes, TestIndexingPerformance, BlockJoinFacetRandomTest, PreAnalyzedUpdateProcessorTest, TestCloudPseudoReturnFields, CloudExitableDirectoryReaderTest, NotRequiredUniqueKeyTest, RemoteQueryErrorTest, TestMergePolicyConfig, TestSolrQueryParser, TestConfigSetsAPI, UUIDFieldTest, SpellPossibilityIteratorTest, PreAnalyzedFieldTest, TestBadConfig, TestTrieFacet, CollectionStateFormat2Test, TestFieldCacheSortRandom, TestCustomDocTransformer, RegexBoostProcessorTest, TestQueryUtils, TestInfoStreamLogging, TestPerFieldSimilarityWithDefaultOverride, TestSolrQueryParserResource, SpatialRPTFieldTypeTest, TestIntervalFaceting, HdfsUnloadDistributedZkTest, SharedFSAutoReplicaFailoverTest, DistanceUnitsTest, DistribDocExpirationUpdateProcessorTest, TestJavabinTupleStreamParser, TestRandomFaceting, DistributedFacetPivotLargeTest, DateMathParserTest, HighlighterMaxOffsetTest, HLLSerializationTest, ReturnFieldsTest, TestConfigReload, HdfsChaosMonkeyNothingIsSafeTest, TestLeaderElectionZkExpiry, TestClusterProperties, BasicDistributedZk2Test, TestOrdValues, UtilsToolTest, TestRebalanceLeaders, EchoParamsTest, TestAuthenticationFramework, TestXmlQParserPlugin, TestWriterPerf, DistributedIntervalFacetingTest, TestDFRSimilarityFactory, TestFieldCache, DistributedTermsComponentTest, BufferStoreTest, CollectionsAPIDistributedZkTest, TestCloudInspectUtil, HdfsBasicDistributedZk2Test, TestRawResponseWriter, TestStandardQParsers, TestUseDocValuesAsStored2, TestGeoJSONResponseWriter, SolrCoreCheckLockOnStartupTest, SuggesterTest, OpenCloseCoreStressTest, SolrSlf4jReporterTest, TestRandomFlRTGCloud, CollectionsAPIAsyncDistributedZkTest, CloneFieldUpdateProcessorFactoryTest, TestPHPSerializedResponseWriter, DistributedExpandComponentTest, QueryParsingTest, TestReplicationHandlerBackup, TestSimpleTrackingShardHandler, TestCustomSort, SimpleCollectionCreateDeleteTest, SolrIndexConfigTest, TestSuggestSpellingConverter, SolrCmdDistributorTest, SpellingQueryConverterTest, SparseHLLTest, V2ApiIntegrationTest, CopyFieldTest, TestHashQParserPlugin, HighlighterConfigTest, TestSubQueryTransformer, HdfsLockFactoryTest, TestImpersonationWithHadoopAuth, TestRecoveryHdfs, FieldAnalysisRequestHandlerTest, TestFoldingMultitermQuery, TestCoreAdminApis, FileBasedSpellCheckerTest, CurrencyFieldOpenExchangeTest, QueryResultKeyTest, TestSearchPerf, TestExpandComponent, TestRequestForwarding, AnalysisErrorHandlingTest, AtomicUpdatesTest, TestSearcherReuse, DistanceFunctionTest, DataDrivenBlockJoinTest, StatsReloadRaceTest, DefaultValueUpdateProcessorTest, ClusterStateTest, BasicZkTest, DocValuesMissingTest, InfoHandlerTest, DeleteLastCustomShardedReplicaTest, TestObjectReleaseTracker, TestConfig, TestSolrCloudWithDelegationTokens, ChangedSchemaMergeTest, SuggestComponentTest, DirectUpdateHandlerOptimizeTest, SliceStateTest, CacheHeaderTest, TestShortCircuitedRequests, TestSolrDeletionPolicy2, OverseerModifyCollectionTest, SolrMetricManagerTest, LukeRequestHandlerTest, TestSolrQueryResponse, SolrMetricsIntegrationTest, TestSystemIdResolver, SolrCloudExampleTest, ManagedSchemaRoundRobinCloudTest, FacetPivotSmallTest, UpdateParamsTest, TestRecovery, ScriptEngineTest, ChaosMonkeySafeLeaderTest, CustomCollectionTest, DistributedFacetPivotSmallAdvancedTest, TestJoin, TestFastWriter, TestAtomicUpdateErrorCases, TestSQLHandlerNonCloud, TestSha256AuthenticationProvider, TestCloudSchemaless, DebugComponentTest, TestLocalFSCloudBackupRestore, LoggingHandlerTest, TestMiniSolrCloudCluster, CoreAdminRequestStatusTest, TemplateUpdateProcessorTest, LeaderInitiatedRecoveryOnCommitTest, TestSolrCoreParser, TestSolrCLIRunExample, TestOmitPositions, CircularListTest, TestJsonFacets, TestExactSharedStatsCache, CloudMLTQParserTest, ClassificationUpdateProcessorIntegrationTest, TestCoreDiscovery, TestUtils, TestQuerySenderNoQuery, NoCacheHeaderTest, ConcurrentDeleteAndCreateCollectionTest, TestNonDefinedSimilarityFactory, BasicDistributedZkTest, OverseerTest, LeaderElectionIntegrationTest, RecoveryZkTest, TestReplicationHandler, ClusterStateUpdateTest, ZkSolrClientTest, TestZkChroot, ZkCLITest, ShardRoutingCustomTest, AutoCommitTest, TestDistributedGrouping, TestHashPartitioner, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestStressVersions, TestRangeQuery, SimpleFacetsTest, StatsComponentTest, QueryElevationComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFiltering, TestFunctionQuery, DirectUpdateHandlerTest, SoftAutoCommitTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, SimplePostToolTest, AnalysisAfterCoreReloadTest, SpellCheckCollatorTest, SuggesterTSTTest, TestStressLucene, TestTrie, SpatialFilterTest, WordBreakSolrSpellCheckerTest, TestUpdate, QueryEqualityTest, SortByFunctionTest, SolrInfoMBeanTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, TestQueryTypes, XmlUpdateRequestHandlerTest, DocumentBuilderTest, TermVectorComponentTest, IndexSchemaRuntimeFieldTest, TestCollationField, JSONWriterTest, MBeansHandlerTest]
   [junit4] Completed [422/691 (1!)] on J1 in 0.79s, 3 tests, 1 failure <<< FAILURES!

[...truncated 517 lines...]
   [junit4] Suite: org.apache.solr.cloud.ShardSplitTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/init-core-data-001
   [junit4]   2> 2897166 INFO  (SUITE-ShardSplitTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2897166 INFO  (SUITE-ShardSplitTest-seed#[46FAF5795D7DA6FC]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 2897173 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2897173 INFO  (Thread-3412) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2897173 INFO  (Thread-3412) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2897275 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.ZkTestServer start zk server on port:57731
   [junit4]   2> 2897318 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2897323 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 2897346 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2897350 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2897354 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2897357 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2897361 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2897368 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2897373 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2897396 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2897410 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2899091 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/control-001/cores/collection1
   [junit4]   2> 2899094 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2899096 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7793d74d{/,null,AVAILABLE}
   [junit4]   2> 2899098 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@68817941{HTTP/1.1,[http/1.1]}{127.0.0.1:57735}
   [junit4]   2> 2899098 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server Started @2905121ms
   [junit4]   2> 2899098 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/tempDir-001/control/data, hostContext=/, hostPort=57735, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/control-001/cores}
   [junit4]   2> 2899099 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2899099 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2899100 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2899100 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2899100 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-09T23:24:16.492Z
   [junit4]   2> 2899104 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2899104 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/control-001/solr.xml
   [junit4]   2> 2899124 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2899124 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57731/solr
   [junit4]   2> 2899173 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57735_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57735_
   [junit4]   2> 2899175 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57735_    ] o.a.s.c.Overseer Overseer (id=97431234450554884-127.0.0.1:57735_-n_0000000000) starting
   [junit4]   2> 2899191 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57735_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57735_
   [junit4]   2> 2899197 INFO  (zkCallback-3822-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2899353 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57735_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/control-001/cores
   [junit4]   2> 2899353 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57735_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2899357 INFO  (OverseerStateUpdate-97431234450554884-127.0.0.1:57735_-n_0000000000) [n:127.0.0.1:57735_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 2900387 WARN  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2900388 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2900399 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2900458 WARN  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2900459 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 2900491 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 2900491 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/control-001/cores/collection1/data/]
   [junit4]   2> 2900491 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@412f4653
   [junit4]   2> 2900496 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=33, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.53125, noCFSRatio=1.0]
   [junit4]   2> 2900530 WARN  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2900624 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2900624 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2900625 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2900625 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2900626 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=73.9990234375, floorSegmentMB=0.61328125, forceMergeDeletesPctAllowed=4.721519625900001, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3537088720629018
   [junit4]   2> 2900627 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@b26f392[collection1] main]
   [junit4]   2> 2900628 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2900629 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2900629 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2900630 INFO  (searcherExecutor-8203-thread-1-processing-n:127.0.0.1:57735_ x:collection1 c:control_collection) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@b26f392[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2900631 INFO  (coreLoadExecutor-8202-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558899754819125248
   [junit4]   2> 2900642 INFO  (coreZkRegister-8195-thread-1-processing-n:127.0.0.1:57735_ x:collection1 c:control_collection) [n:127.0.0.1:57735_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2900642 INFO  (coreZkRegister-8195-thread-1-processing-n:127.0.0.1:57735_ x:collection1 c:control_collection) [n:127.0.0.1:57735_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2900643 INFO  (coreZkRegister-8195-thread-1-processing-n:127.0.0.1:57735_ x:collection1 c:control_collection) [n:127.0.0.1:57735_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57735/collection1/
   [junit4]   2> 2900643 INFO  (coreZkRegister-8195-thread-1-processing-n:127.0.0.1:57735_ x:collection1 c:control_collection) [n:127.0.0.1:57735_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2900643 INFO  (coreZkRegister-8195-thread-1-processing-n:127.0.0.1:57735_ x:collection1 c:control_collection) [n:127.0.0.1:57735_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:57735/collection1/ has no replicas
   [junit4]   2> 2900653 INFO  (coreZkRegister-8195-thread-1-processing-n:127.0.0.1:57735_ x:collection1 c:control_collection) [n:127.0.0.1:57735_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57735/collection1/ shard1
   [junit4]   2> 2900811 INFO  (coreZkRegister-8195-thread-1-processing-n:127.0.0.1:57735_ x:collection1 c:control_collection) [n:127.0.0.1:57735_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2901098 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2901100 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57731/solr ready
   [junit4]   2> 2901100 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2901513 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-1-001/cores/collection1
   [junit4]   2> 2901515 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-1-001
   [junit4]   2> 2901516 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2901517 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7d6ee70{/,null,AVAILABLE}
   [junit4]   2> 2901518 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@554bc381{HTTP/1.1,[http/1.1]}{127.0.0.1:57740}
   [junit4]   2> 2901518 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server Started @2907540ms
   [junit4]   2> 2901519 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=57740, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-1-001/cores}
   [junit4]   2> 2901519 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2901519 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2901519 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2901519 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2901519 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-09T23:24:18.911Z
   [junit4]   2> 2901524 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2901525 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-1-001/solr.xml
   [junit4]   2> 2901558 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2901559 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57731/solr
   [junit4]   2> 2901583 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57740_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2901594 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57740_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57740_
   [junit4]   2> 2901599 INFO  (zkCallback-3826-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2901600 INFO  (zkCallback-3822-thread-2-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2901601 INFO  (zkCallback-3831-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2901759 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57740_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-1-001/cores
   [junit4]   2> 2901759 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57740_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2901762 INFO  (OverseerStateUpdate-97431234450554884-127.0.0.1:57735_-n_0000000000) [n:127.0.0.1:57735_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 2902792 WARN  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2902793 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2902806 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2902862 WARN  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2902863 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 2902893 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2902893 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 2902893 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@412f4653
   [junit4]   2> 2902897 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=33, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.53125, noCFSRatio=1.0]
   [junit4]   2> 2902925 WARN  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2903047 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2903047 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2903049 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2903049 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2903051 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=73.9990234375, floorSegmentMB=0.61328125, forceMergeDeletesPctAllowed=4.721519625900001, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3537088720629018
   [junit4]   2> 2903052 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@257dbce5[collection1] main]
   [junit4]   2> 2903054 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2903055 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2903055 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2903057 INFO  (searcherExecutor-8214-thread-1-processing-n:127.0.0.1:57740_ x:collection1 c:collection1) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@257dbce5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2903058 INFO  (coreLoadExecutor-8213-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558899757364019200
   [junit4]   2> 2903081 INFO  (coreZkRegister-8208-thread-1-processing-n:127.0.0.1:57740_ x:collection1 c:collection1) [n:127.0.0.1:57740_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2903081 INFO  (coreZkRegister-8208-thread-1-processing-n:127.0.0.1:57740_ x:collection1 c:collection1) [n:127.0.0.1:57740_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2903081 INFO  (coreZkRegister-8208-thread-1-processing-n:127.0.0.1:57740_ x:collection1 c:collection1) [n:127.0.0.1:57740_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57740/collection1/
   [junit4]   2> 2903081 INFO  (coreZkRegister-8208-thread-1-processing-n:127.0.0.1:57740_ x:collection1 c:collection1) [n:127.0.0.1:57740_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2903081 INFO  (coreZkRegister-8208-thread-1-processing-n:127.0.0.1:57740_ x:collection1 c:collection1) [n:127.0.0.1:57740_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:57740/collection1/ has no replicas
   [junit4]   2> 2903088 INFO  (coreZkRegister-8208-thread-1-processing-n:127.0.0.1:57740_ x:collection1 c:collection1) [n:127.0.0.1:57740_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57740/collection1/ shard2
   [junit4]   2> 2903254 INFO  (coreZkRegister-8208-thread-1-processing-n:127.0.0.1:57740_ x:collection1 c:collection1) [n:127.0.0.1:57740_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2903983 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-2-001/cores/collection1
   [junit4]   2> 2903987 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-2-001
   [junit4]   2> 2903989 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2903991 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6aad9875{/,null,AVAILABLE}
   [junit4]   2> 2903992 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7cd63fcd{HTTP/1.1,[http/1.1]}{127.0.0.1:57744}
   [junit4]   2> 2903992 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server Started @2910014ms
   [junit4]   2> 2903992 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=57744, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-2-001/cores}
   [junit4]   2> 2903993 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2903993 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2903993 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2903993 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2903993 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-09T23:24:21.385Z
   [junit4]   2> 2903999 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2903999 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-2-001/solr.xml
   [junit4]   2> 2904016 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2904017 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57731/solr
   [junit4]   2> 2904037 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57744_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2904048 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57744_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57744_
   [junit4]   2> 2904052 INFO  (zkCallback-3822-thread-3-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2904052 INFO  (zkCallback-3831-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2904052 INFO  (zkCallback-3826-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2904054 INFO  (zkCallback-3837-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2904229 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57744_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-2-001/cores
   [junit4]   2> 2904229 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57744_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2904232 INFO  (OverseerStateUpdate-97431234450554884-127.0.0.1:57735_-n_0000000000) [n:127.0.0.1:57735_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 2905253 WARN  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2905254 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2905272 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2905358 WARN  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2905360 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 2905393 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2905393 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 2905393 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@412f4653
   [junit4]   2> 2905406 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=33, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.53125, noCFSRatio=1.0]
   [junit4]   2> 2905433 WARN  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2905988 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2905988 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2905989 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2905990 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2905991 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=73.9990234375, floorSegmentMB=0.61328125, forceMergeDeletesPctAllowed=4.721519625900001, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3537088720629018
   [junit4]   2> 2905992 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5937f46a[collection1] main]
   [junit4]   2> 2905993 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2905994 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2905994 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2905996 INFO  (searcherExecutor-8225-thread-1-processing-n:127.0.0.1:57744_ x:collection1 c:collection1) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5937f46a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2905996 INFO  (coreLoadExecutor-8224-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558899760444735488
   [junit4]   2> 2906007 INFO  (coreZkRegister-8219-thread-1-processing-n:127.0.0.1:57744_ x:collection1 c:collection1) [n:127.0.0.1:57744_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2906007 INFO  (coreZkRegister-8219-thread-1-processing-n:127.0.0.1:57744_ x:collection1 c:collection1) [n:127.0.0.1:57744_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2906007 INFO  (coreZkRegister-8219-thread-1-processing-n:127.0.0.1:57744_ x:collection1 c:collection1) [n:127.0.0.1:57744_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57744/collection1/
   [junit4]   2> 2906007 INFO  (coreZkRegister-8219-thread-1-processing-n:127.0.0.1:57744_ x:collection1 c:collection1) [n:127.0.0.1:57744_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2906007 INFO  (coreZkRegister-8219-thread-1-processing-n:127.0.0.1:57744_ x:collection1 c:collection1) [n:127.0.0.1:57744_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:57744/collection1/ has no replicas
   [junit4]   2> 2906015 INFO  (coreZkRegister-8219-thread-1-processing-n:127.0.0.1:57744_ x:collection1 c:collection1) [n:127.0.0.1:57744_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57744/collection1/ shard1
   [junit4]   2> 2906181 INFO  (coreZkRegister-8219-thread-1-processing-n:127.0.0.1:57744_ x:collection1 c:collection1) [n:127.0.0.1:57744_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2907079 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-3-001/cores/collection1
   [junit4]   2> 2907087 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-3-001
   [junit4]   2> 2907087 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2907089 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@13f05ee{/,null,AVAILABLE}
   [junit4]   2> 2907090 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@629f90cb{HTTP/1.1,[http/1.1]}{127.0.0.1:57748}
   [junit4]   2> 2907090 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server Started @2913112ms
   [junit4]   2> 2907090 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=57748, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-3-001/cores}
   [junit4]   2> 2907091 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2907091 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2907092 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2907092 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2907092 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-09T23:24:24.484Z
   [junit4]   2> 2907096 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2907096 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-3-001/solr.xml
   [junit4]   2> 2907104 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2907105 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57731/solr
   [junit4]   2> 2907122 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57748_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 2907133 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57748_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57748_
   [junit4]   2> 2907140 INFO  (zkCallback-3831-thread-1-processing-n:127.0.0.1:57740_) [n:127.0.0.1:57740_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2907142 INFO  (zkCallback-3843-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2907142 INFO  (zkCallback-3822-thread-1-processing-n:127.0.0.1:57735_) [n:127.0.0.1:57735_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2907142 INFO  (zkCallback-3826-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2907142 INFO  (zkCallback-3837-thread-1-processing-n:127.0.0.1:57744_) [n:127.0.0.1:57744_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2907232 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57748_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-3-001/cores
   [junit4]   2> 2907232 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [n:127.0.0.1:57748_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2907239 INFO  (OverseerStateUpdate-97431234450554884-127.0.0.1:57735_-n_0000000000) [n:127.0.0.1:57735_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 2908262 WARN  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2908262 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2908274 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2908325 WARN  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2908326 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 2908359 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2908359 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-3-001/cores/collection1/data/]
   [junit4]   2> 2908360 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@412f4653
   [junit4]   2> 2908363 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=33, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.53125, noCFSRatio=1.0]
   [junit4]   2> 2908376 WARN  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2908490 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2908490 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2908491 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2908491 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2908492 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=73.9990234375, floorSegmentMB=0.61328125, forceMergeDeletesPctAllowed=4.721519625900001, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3537088720629018
   [junit4]   2> 2908493 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5b5ee7e[collection1] main]
   [junit4]   2> 2908495 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2908496 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2908496 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2908497 INFO  (searcherExecutor-8236-thread-1-processing-n:127.0.0.1:57748_ x:collection1 c:collection1) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5b5ee7e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2908497 INFO  (coreLoadExecutor-8235-thread-1-processing-n:127.0.0.1:57748_) [n:127.0.0.1:57748_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558899763067224064
   [junit4]   2> 2908502 INFO  (coreZkRegister-8230-thread-1-processing-n:127.0.0.1:57748_ x:collection1 c:collection1) [n:127.0.0.1:57748_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 2908503 INFO  (updateExecutor-3840-thread-1-processing-n:127.0.0.1:57748_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:57748_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 2908503 INFO  (recoveryExecutor-3841-thread-1-processing-n:127.0.0.1:57748_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:57748_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 2908503 INFO  (recoveryExecutor-3841-thread-1-processing-n:127.0.0.1:57748_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:57748_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 2908503 INFO  (recoveryExecutor-3841-thread-1-processing-n:127.0.0.1:57748_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:57748_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
   [junit4]   2> 2908503 INFO  (recoveryExecutor-3841-thread-1-processing-n:127.0.0.1:57748_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:57748_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 2908504 INFO  (recoveryExecutor-3841-thread-1-processing-n:127.0.0.1:57748_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:57748_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:57740/collection1/] and I am [http://127.0.0.1:57748/collection1/]
   [junit4]   2> 2908508 INFO  (recoveryExecutor-3841-thread-1-processing-n:127.0.0.1:57748_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:57748_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:57740]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:57748_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 2908511 INFO  (qtp267660466-17962) [n:127.0.0.1:57740_    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 2908512 INFO  (qtp267660466-17962) [n:127.0.0.1:57740_    ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard2 of collection1) have state: recovering
   [junit4]   2> 2908512 INFO  (qtp267660466-17962) [n:127.0.0.1:57740_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:57748_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:57748","node_name":"127.0.0.1:57748_","state":"down"}
   [junit4]   2> 2909467 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-4-001/cores/collection1
   [junit4]   2> 2909473 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.ShardSplitTest_46FAF5795D7DA6FC-001/shard-4-001
   [junit4]   2> 2909474 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2909476 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3eb25cf2{/,null,AVAILABLE}
   [junit4]   2> 2909477 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7440bea8{HTTP/1.1,[http/1.1]}{127.0.0.1:57753}
   [junit4]   2> 2909478 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[46FAF5795D7DA6FC]) [    ] o.e.j.s.Server 

[...truncated too long message...]

closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 22. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java (at line 51)
 [ecj-lint] 	new JavaBinCodec(resolver).setWritableDocFields(resolver).marshal(response.getValues(), out);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] 23. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java (at line 163)
 [ecj-lint] 	new JavaBinCodec(resolver).setWritableDocFields(resolver).marshal(rsp.getValues(), out);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] 24. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java (at line 166)
 [ecj-lint] 	return (NamedList<Object>) new JavaBinCodec(resolver).unmarshal(in);
 [ecj-lint] 	                           ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 25. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/response/SmileResponseWriter.java (at line 33)
 [ecj-lint] 	new SmileWriter(out, request, response).writeResponse();
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 26. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java (at line 146)
 [ecj-lint] 	ratesJsonStream = resourceLoader.openResource(ratesFileLocation);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'ratesJsonStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 27. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (at line 1185)
 [ecj-lint] 	UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField), mapping);
 [ecj-lint] 	                  ^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'uninvertingReader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 28. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/search/CursorMark.java (at line 188)
 [ecj-lint] 	pieces = (List<Object>) new JavaBinCodec().unmarshal(in);
 [ecj-lint] 	                        ^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] 29. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/search/CursorMark.java (at line 265)
 [ecj-lint] 	new JavaBinCodec().marshal(marshalledValues, out);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 30. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 145)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 31. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 154)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 32. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 163)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 33. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java (at line 78)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 34. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java (at line 78)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 35. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (at line 263)
 [ecj-lint] 	throw new SolrException(ErrorCode.SERVER_ERROR, "Error occurred while loading solr.xml from zookeeper", e);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'loader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 36. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java (at line 98)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 37. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 188)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 38. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 258)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 39. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 226)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 40. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 226)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 41. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 258)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 42. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 273)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 43. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 492)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 44. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] 45. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 46. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (at line 149)
 [ecj-lint] 	SolrQueryRequest req = requestInfo == null ? null : requestInfo.getReq();
 [ecj-lint] 	                 ^^^
 [ecj-lint] Resource leak: 'req' is never closed
 [ecj-lint] ----------
 [ecj-lint] 46 problems (2 errors, 44 warnings)

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:775: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:674: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/common-build.xml:2001: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/common-build.xml:2040: Compile failed; see the compiler error output for details.

Total time: 114 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 693 - Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/693/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:
Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>

Stack Trace:
java.lang.AssertionError: Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
	at __randomizedtesting.SeedInfo.seed([3CF871AE4C014DE1:B4AC4E74E2FD2019]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotSame(Assert.java:641)
	at org.junit.Assert.assertSame(Assert.java:580)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:250)
	at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReqParamsAPI.test

Error Message:
Could not get expected value  'A val' for path 'params/a' full output: {   "responseHeader":{     "status":0,     "QTime":0},   "params":{     "wt":"json",     "useParams":""},   "context":{     "webapp":"/solr",     "path":"/dump0",     "httpMethod":"GET"}},  from server:  http://127.0.0.1:54239/solr/collection1_shard1_replica1

Stack Trace:
java.lang.AssertionError: Could not get expected value  'A val' for path 'params/a' full output: {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "params":{
    "wt":"json",
    "useParams":""},
  "context":{
    "webapp":"/solr",
    "path":"/dump0",
    "httpMethod":"GET"}},  from server:  http://127.0.0.1:54239/solr/collection1_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([3CF871AE4C014DE1:B4AC4E74E2FD2019]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:556)
	at org.apache.solr.handler.TestReqParamsAPI.testReqParams(TestReqParamsAPI.java:127)
	at org.apache.solr.handler.TestReqParamsAPI.test(TestReqParamsAPI.java:69)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11058 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReqParamsAPI
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/init-core-data-001
   [junit4]   2> 290558 INFO  (SUITE-TestReqParamsAPI-seed#[3CF871AE4C014DE1]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 290559 INFO  (SUITE-TestReqParamsAPI-seed#[3CF871AE4C014DE1]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001
   [junit4]   2> 290559 INFO  (SUITE-TestReqParamsAPI-seed#[3CF871AE4C014DE1]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 290559 INFO  (Thread-305) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 290560 INFO  (Thread-305) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 290669 INFO  (SUITE-TestReqParamsAPI-seed#[3CF871AE4C014DE1]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:54237
   [junit4]   2> 290691 INFO  (jetty-launcher-382-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 290691 INFO  (jetty-launcher-382-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 290695 INFO  (jetty-launcher-382-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3718130f{/solr,null,AVAILABLE}
   [junit4]   2> 290695 INFO  (jetty-launcher-382-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3684cc20{/solr,null,AVAILABLE}
   [junit4]   2> 290696 INFO  (jetty-launcher-382-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7666374b{HTTP/1.1,[http/1.1]}{127.0.0.1:54239}
   [junit4]   2> 290697 INFO  (jetty-launcher-382-thread-1) [    ] o.e.j.s.Server Started @295842ms
   [junit4]   2> 290697 INFO  (jetty-launcher-382-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54239}
   [junit4]   2> 290697 ERROR (jetty-launcher-382-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 290697 INFO  (jetty-launcher-382-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@62849540{HTTP/1.1,[http/1.1]}{127.0.0.1:54240}
   [junit4]   2> 290697 INFO  (jetty-launcher-382-thread-2) [    ] o.e.j.s.Server Started @295843ms
   [junit4]   2> 290698 INFO  (jetty-launcher-382-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54240}
   [junit4]   2> 290698 ERROR (jetty-launcher-382-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 290698 INFO  (jetty-launcher-382-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 290698 INFO  (jetty-launcher-382-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 290698 INFO  (jetty-launcher-382-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 290698 INFO  (jetty-launcher-382-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-13T01:47:54.934Z
   [junit4]   2> 290699 INFO  (jetty-launcher-382-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 290700 INFO  (jetty-launcher-382-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 290700 INFO  (jetty-launcher-382-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 290700 INFO  (jetty-launcher-382-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-13T01:47:54.936Z
   [junit4]   2> 290707 INFO  (jetty-launcher-382-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 290708 INFO  (jetty-launcher-382-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 290720 INFO  (jetty-launcher-382-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 290720 INFO  (jetty-launcher-382-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 290721 INFO  (jetty-launcher-382-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54237/solr
   [junit4]   2> 290722 INFO  (jetty-launcher-382-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54237/solr
   [junit4]   2> 290778 INFO  (jetty-launcher-382-thread-1) [n:127.0.0.1:54239_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 290778 INFO  (jetty-launcher-382-thread-2) [n:127.0.0.1:54240_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 290779 INFO  (jetty-launcher-382-thread-1) [n:127.0.0.1:54239_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54239_solr
   [junit4]   2> 290781 INFO  (jetty-launcher-382-thread-2) [n:127.0.0.1:54240_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54240_solr
   [junit4]   2> 290782 INFO  (jetty-launcher-382-thread-1) [n:127.0.0.1:54239_solr    ] o.a.s.c.Overseer Overseer (id=97448786316361734-127.0.0.1:54239_solr-n_0000000000) starting
   [junit4]   2> 290792 INFO  (zkCallback-392-thread-1-processing-n:127.0.0.1:54240_solr) [n:127.0.0.1:54240_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 290792 INFO  (zkCallback-393-thread-1-processing-n:127.0.0.1:54239_solr) [n:127.0.0.1:54239_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 290826 INFO  (jetty-launcher-382-thread-1) [n:127.0.0.1:54239_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54239_solr
   [junit4]   2> 290847 INFO  (zkCallback-393-thread-1-processing-n:127.0.0.1:54239_solr) [n:127.0.0.1:54239_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 290848 INFO  (zkCallback-392-thread-1-processing-n:127.0.0.1:54240_solr) [n:127.0.0.1:54240_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 290909 INFO  (jetty-launcher-382-thread-2) [n:127.0.0.1:54240_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/.
   [junit4]   2> 290931 INFO  (jetty-launcher-382-thread-1) [n:127.0.0.1:54239_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node1/.
   [junit4]   2> 290979 INFO  (SUITE-TestReqParamsAPI-seed#[3CF871AE4C014DE1]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 290982 INFO  (SUITE-TestReqParamsAPI-seed#[3CF871AE4C014DE1]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54237/solr ready
   [junit4]   2> 291038 INFO  (qtp667814353-1718) [n:127.0.0.1:54240_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf1&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 291042 INFO  (OverseerThreadFactory-501-thread-1-processing-n:127.0.0.1:54239_solr) [n:127.0.0.1:54239_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 291166 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=collection1_shard1_replica2&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 291166 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 291289 INFO  (zkCallback-393-thread-1-processing-n:127.0.0.1:54239_solr) [n:127.0.0.1:54239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 291290 INFO  (zkCallback-392-thread-1-processing-n:127.0.0.1:54240_solr) [n:127.0.0.1:54240_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 292190 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 292190 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 292253 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.s.IndexSchema [collection1_shard1_replica2] Schema name=minimal
   [junit4]   2> 292254 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 292258 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 292258 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 292260 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica2' using configuration from collection collection1
   [junit4]   2> 292260 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 292261 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.SolrCore [[collection1_shard1_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/collection1_shard1_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/./collection1_shard1_replica2/data/]
   [junit4]   2> 292261 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 292339 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 292339 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 292339 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 292339 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 292341 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 292341 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 292341 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 292341 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 292342 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@8949e3f[collection1_shard1_replica1] main]
   [junit4]   2> 292342 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@3c98d8a8[collection1_shard1_replica2] main]
   [junit4]   2> 292344 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 292344 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 292345 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 292345 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 292364 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 292364 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 292374 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 292374 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 292374 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 292374 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 292374 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559180582961807360
   [junit4]   2> 292374 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559180582961807360
   [junit4]   2> 292376 INFO  (searcherExecutor-506-thread-1-processing-n:127.0.0.1:54240_solr x:collection1_shard1_replica2 s:shard1 c:collection1) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2] Registered new searcher Searcher@3c98d8a8[collection1_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 292376 INFO  (searcherExecutor-507-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@8949e3f[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 292384 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 292384 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 292384 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54240/solr/collection1_shard1_replica2/
   [junit4]   2> 292385 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica2 url=http://127.0.0.1:54240/solr START replicas=[http://127.0.0.1:54239/solr/collection1_shard1_replica1/] nUpdates=100
   [junit4]   2> 292385 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica2 url=http://127.0.0.1:54240/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 292388 INFO  (qtp1697556344-1725) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.S.Request [collection1_shard1_replica1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 292389 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 292389 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 292395 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54240/solr/collection1_shard1_replica2/ shard1
   [junit4]   2> 292509 INFO  (zkCallback-392-thread-1-processing-n:127.0.0.1:54240_solr) [n:127.0.0.1:54240_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 292509 INFO  (zkCallback-393-thread-2-processing-n:127.0.0.1:54239_solr) [n:127.0.0.1:54239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 292564 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 292567 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1  x:collection1_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=collection1_shard1_replica2&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=1400
   [junit4]   2> 292675 INFO  (zkCallback-393-thread-1-processing-n:127.0.0.1:54239_solr) [n:127.0.0.1:54239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 292675 INFO  (zkCallback-392-thread-1-processing-n:127.0.0.1:54240_solr) [n:127.0.0.1:54240_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 293390 INFO  (qtp1697556344-1723) [n:127.0.0.1:54239_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=2223
   [junit4]   2> 293394 INFO  (qtp667814353-1718) [n:127.0.0.1:54240_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 293501 INFO  (zkCallback-393-thread-2-processing-n:127.0.0.1:54239_solr) [n:127.0.0.1:54239_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 293501 INFO  (zkCallback-392-thread-1-processing-n:127.0.0.1:54240_solr) [n:127.0.0.1:54240_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 294398 INFO  (qtp667814353-1718) [n:127.0.0.1:54240_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf1&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=3360
   [junit4]   2> 294417 INFO  (TEST-TestReqParamsAPI.test-seed#[3CF871AE4C014DE1]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 294476 INFO  (TEST-TestReqParamsAPI.test-seed#[3CF871AE4C014DE1]) [    ] o.a.s.c.TestSolrConfigHandler going to send config command. path /config?wt=json , payload: {
   [junit4]   2> 'create-requesthandler' : { 'name' : '/dump0', 'class': 'org.apache.solr.handler.DumpRequestHandler' }
   [junit4]   2> }
   [junit4]   2> 294519 INFO  (qtp1697556344-1727) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Executed config commands successfully and persisted to ZK [{"create-requesthandler":{
   [junit4]   2>     "name":"/dump0",
   [junit4]   2>     "class":"org.apache.solr.handler.DumpRequestHandler"}}]
   [junit4]   2> 294521 INFO  (Thread-324) [n:127.0.0.1:54240_solr    ] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica2
   [junit4]   2> 294522 INFO  (Thread-325) [n:127.0.0.1:54239_solr    ] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica1
   [junit4]   2> 294523 INFO  (Thread-324) [n:127.0.0.1:54240_solr    ] o.a.s.c.SolrCore core reload collection1_shard1_replica2
   [junit4]   2> 294523 INFO  (Thread-325) [n:127.0.0.1:54239_solr    ] o.a.s.c.SolrCore core reload collection1_shard1_replica1
   [junit4]   2> 294526 INFO  (qtp1697556344-1727) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Waiting up to 30 secs for 2 replicas to set the property overlay to be of version 0 for collection collection1
   [junit4]   2> 294528 INFO  (solrHandlerExecutor-516-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 294529 INFO  (solrHandlerExecutor-516-thread-2-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 294539 INFO  (Thread-324) [n:127.0.0.1:54240_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 294539 INFO  (Thread-325) [n:127.0.0.1:54239_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 294548 INFO  (Thread-324) [n:127.0.0.1:54240_solr    ] o.a.s.s.IndexSchema [collection1_shard1_replica2] Schema name=minimal
   [junit4]   2> 294548 INFO  (Thread-325) [n:127.0.0.1:54239_solr    ] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 294552 INFO  (Thread-324) [n:127.0.0.1:54240_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 294554 INFO  (Thread-325) [n:127.0.0.1:54239_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 294558 INFO  (Thread-324) [n:127.0.0.1:54240_solr    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1_shard1_replica2' using configuration from collection collection1
   [junit4]   2> 294560 INFO  (Thread-325) [n:127.0.0.1:54239_solr    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 294560 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [[collection1_shard1_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/collection1_shard1_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/./collection1_shard1_replica2/data/]
   [junit4]   2> 294571 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 294634 INFO  (qtp667814353-1721) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 294635 INFO  (qtp667814353-1721) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.S.Request [collection1_shard1_replica2]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 294635 INFO  (qtp1697556344-1729) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 294636 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler Trying to update my configs
   [junit4]   2> 294636 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica2
   [junit4]   2> 294636 INFO  (qtp1697556344-1729) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.S.Request [collection1_shard1_replica1]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 294637 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Trying to update my configs
   [junit4]   2> 294637 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica1
   [junit4]   2> 294638 INFO  (solrHandlerExecutor-516-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54239/solr/collection1_shard1_replica1/ for prop overlay   after 1 attempts
   [junit4]   2> 294638 INFO  (solrHandlerExecutor-516-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 294639 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore core reload collection1_shard1_replica2
   [junit4]   2> 294639 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore core reload collection1_shard1_replica1
   [junit4]   2> 294640 INFO  (solrHandlerExecutor-516-thread-2-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54240/solr/collection1_shard1_replica2/ for prop overlay   after 1 attempts
   [junit4]   2> 294640 INFO  (solrHandlerExecutor-516-thread-2-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 294770 INFO  (qtp1697556344-1716) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 294770 INFO  (qtp1697556344-1716) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.S.Request [collection1_shard1_replica1]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 294771 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Another reload is in progress . Not doing anything
   [junit4]   2> 294771 INFO  (solrHandlerExecutor-516-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54239/solr/collection1_shard1_replica1/ for prop overlay   after 2 attempts
   [junit4]   2> 294771 INFO  (solrHandlerExecutor-516-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 294783 INFO  (qtp667814353-1728) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 294784 INFO  (qtp667814353-1728) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.S.Request [collection1_shard1_replica2]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 294784 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler Another reload is in progress . Not doing anything
   [junit4]   2> 294784 INFO  (solrHandlerExecutor-516-thread-2-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54240/solr/collection1_shard1_replica2/ for prop overlay   after 2 attempts
   [junit4]   2> 294784 INFO  (solrHandlerExecutor-516-thread-2-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 294792 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 294792 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 294793 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@48d441fd[collection1_shard1_replica2] main]
   [junit4]   2> 294795 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 294796 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 294796 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 294797 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 294797 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 294798 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559180585503555584
   [junit4]   2> 294798 INFO  (searcherExecutor-517-thread-1-processing-n:127.0.0.1:54240_solr x:collection1_shard1_replica2 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2] Registered new searcher Searcher@48d441fd[collection1_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 294821 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 294821 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 294849 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@637c4096[collection1_shard1_replica1] main]
   [junit4]   2> 294850 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 294851 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@1c10935b[collection1_shard1_replica2] main]
   [junit4]   2> 294851 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@2e1592e2
   [junit4]   2> 294851 INFO  (searcherExecutor-517-thread-1-processing-n:127.0.0.1:54240_solr x:collection1_shard1_replica2 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2] Registered new searcher Searcher@1c10935b[collection1_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 294852 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 294852 INFO  (Thread-324) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1.shard1.replica2
   [junit4]   2> 294852 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 294852 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 294854 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 294855 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 294855 INFO  (searcherExecutor-518-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@637c4096[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 294855 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559180585563324416
   [junit4]   2> 294864 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 294873 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.IndexSchema [collection1_shard1_replica2] Schema name=minimal
   [junit4]   2> 294878 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 294880 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.CoreContainer Reloading SolrCore 'collection1_shard1_replica2' using configuration from collection collection1
   [junit4]   2> 294880 INFO  (qtp1697556344-1724) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler expecting overlay version 0 but my version is -1
   [junit4]   2> 294881 INFO  (qtp1697556344-1724) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.S.Request [collection1_shard1_replica1]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 294881 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Another reload is in progress . Not doing anything
   [junit4]   2> 294881 INFO  (solrHandlerExecutor-516-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 0 from http://127.0.0.1:54239/solr/collection1_shard1_replica1/ for prop overlay   after 3 attempts
   [junit4]   2> 294881 INFO  (solrHandlerExecutor-516-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 294890 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [[collection1_shard1_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/collection1_shard1_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/./collection1_shard1_replica2/data/]
   [junit4]   2> 294903 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 294903 INFO  (Thread-325) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@33f1ce86[collection1_shard1_replica1] main]
   [junit4]   2> 294905 INFO  (searcherExecutor-518-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@33f1ce86[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 294909 INFO  (qtp667814353-1730) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler I already have the expected version 0 of config
   [junit4]   2> 294909 INFO  (qtp667814353-1730) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler isStale false , resourceloader org.apache.solr.cloud.ZkSolrResourceLoader
   [junit4]   2> 294909 INFO  (qtp667814353-1730) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.S.Request [collection1_shard1_replica2]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 294925 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 294932 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 294933 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 294934 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 294935 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 294986 INFO  (qtp1697556344-1725) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler I already have the expected version 0 of config
   [junit4]   2> 294986 INFO  (qtp1697556344-1725) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler isStale false , resourceloader org.apache.solr.cloud.ZkSolrResourceLoader
   [junit4]   2> 294986 INFO  (qtp1697556344-1725) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.S.Request [collection1_shard1_replica1]  webapp=/solr path=/config/znodeVersion params={overlay=0&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 294987 INFO  (qtp1697556344-1727) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Took 468.0ms to set the property overlay to be of version 0 for collection collection1
   [junit4]   2> 294988 INFO  (qtp1697556344-1727) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.S.Request [collection1_shard1_replica1]  webapp=/solr path=/config params={wt=json} status=0 QTime=508
   [junit4]   2> 294988 INFO  (qtp1697556344-1727) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4df8a616
   [junit4]   2> 294988 INFO  (qtp1697556344-1727) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1.shard1.replica1
   [junit4]   2> 294989 INFO  (TEST-TestReqParamsAPI.test-seed#[3CF871AE4C014DE1]) [    ] o.a.s.c.TestSolrConfigHandler going to send config command. path /config?wt=json , payload: {
   [junit4]   2> 'create-requesthandler' : { 'name' : '/dump1', 'class': 'org.apache.solr.handler.DumpRequestHandler', 'useParams':'x' }
   [junit4]   2> }
   [junit4]   2> 294993 INFO  (qtp1697556344-1726) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ZkController Persisted config data to node /configs/conf1/configoverlay.json 
   [junit4]   2> 294993 INFO  (qtp1697556344-1726) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Executed config commands successfully and persisted to ZK [{"create-requesthandler":{
   [junit4]   2>     "name":"/dump1",
   [junit4]   2>     "class":"org.apache.solr.handler.DumpRequestHandler",
   [junit4]   2>     "useParams":"x"}}]
   [junit4]   2> 294994 INFO  (qtp1697556344-1726) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Waiting up to 30 secs for 2 replicas to set the property overlay to be of version 1 for collection collection1
   [junit4]   2> 294995 INFO  (Thread-327) [n:127.0.0.1:54240_solr    ] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica2
   [junit4]   2> 294995 INFO  (Thread-326) [n:127.0.0.1:54239_solr    ] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica1
   [junit4]   2> 294997 INFO  (Thread-327) [n:127.0.0.1:54240_solr    ] o.a.s.c.SolrCore core reload collection1_shard1_replica2
   [junit4]   2> 294997 INFO  (Thread-326) [n:127.0.0.1:54239_solr    ] o.a.s.c.SolrCore core reload collection1_shard1_replica1
   [junit4]   2> 294997 INFO  (solrHandlerExecutor-527-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 294997 INFO  (solrHandlerExecutor-527-thread-2-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 295028 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 295028 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 295031 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 295031 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 295033 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2e438a2f[collection1_shard1_replica1] main]
   [junit4]   2> 295034 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@186bee85[collection1_shard1_replica2] main]
   [junit4]   2> 295035 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 295036 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 295036 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 295036 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 295036 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 295036 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 295037 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 295038 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 295038 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 295038 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 295038 INFO  (searcherExecutor-525-thread-1-processing-n:127.0.0.1:54240_solr x:collection1_shard1_replica2 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2] Registered new searcher Searcher@186bee85[collection1_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 295038 INFO  (searcherExecutor-526-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@2e438a2f[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 295038 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559180585755213824
   [junit4]   2> 295038 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559180585755213824
   [junit4]   2> 295041 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 295042 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 295042 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6013cd09[collection1_shard1_replica1] main]
   [junit4]   2> 295042 INFO  (searcherExecutor-526-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@6013cd09[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 295052 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@bd64af7[collection1_shard1_replica2] main]
   [junit4]   2> 295053 INFO  (OverseerCollectionConfigSetProcessor-97448786316361734-127.0.0.1:54239_solr-n_0000000000) [n:127.0.0.1:54239_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 295057 INFO  (Thread-326) [n:127.0.0.1:54239_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 295064 INFO  (Thread-326) [n:127.0.0.1:54239_solr    ] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 295064 INFO  (searcherExecutor-525-thread-1-processing-n:127.0.0.1:54240_solr x:collection1_shard1_replica2 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2] Registered new searcher Searcher@bd64af7[collection1_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 295065 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@5fcf2d7b
   [junit4]   2> 295067 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1.shard1.replica2
   [junit4]   2> 295067 INFO  (Thread-326) [n:127.0.0.1:54239_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 295070 INFO  (Thread-326) [n:127.0.0.1:54239_solr    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 295075 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 295076 INFO  (Thread-327) [n:127.0.0.1:54240_solr    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 295085 INFO  (Thread-327) [n:127.0.0.1:54240_solr    ] o.a.s.s.IndexSchema [collection1_shard1_replica2] Schema name=minimal
   [junit4]   2> 295088 INFO  (Thread-327) [n:127.0.0.1:54240_solr    ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 295089 INFO  (Thread-327) [n:127.0.0.1:54240_solr    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1_shard1_replica2' using configuration from collection collection1
   [junit4]   2> 295100 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [[collection1_shard1_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/collection1_shard1_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/./collection1_shard1_replica2/data/]
   [junit4]   2> 295101 INFO  (qtp1697556344-1724) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler expecting overlay version 1 but my version is 0
   [junit4]   2> 295103 INFO  (qtp667814353-1715) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler expecting overlay version 1 but my version is 0
   [junit4]   2> 295103 INFO  (qtp1697556344-1724) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.S.Request [collection1_shard1_replica1]  webapp=/solr path=/config/znodeVersion params={overlay=1&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 295103 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Trying to update my configs
   [junit4]   2> 295103 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica1
   [junit4]   2> 295104 INFO  (qtp667814353-1715) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.S.Request [collection1_shard1_replica2]  webapp=/solr path=/config/znodeVersion params={overlay=1&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 295105 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler Trying to update my configs
   [junit4]   2> 295105 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica2
   [junit4]   2> 295105 INFO  (solrHandlerExecutor-527-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 1 from http://127.0.0.1:54239/solr/collection1_shard1_replica1/ for prop overlay   after 1 attempts
   [junit4]   2> 295105 INFO  (solrHandlerExecutor-527-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 295106 INFO  (solrHandlerExecutor-527-thread-2-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Could not get expectedVersion 1 from http://127.0.0.1:54240/solr/collection1_shard1_replica2/ for prop overlay   after 1 attempts
   [junit4]   2> 295106 INFO  (solrHandlerExecutor-527-thread-2-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 295107 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore core reload collection1_shard1_replica1
   [junit4]   2> 295108 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore core reload collection1_shard1_replica2
   [junit4]   2> 295163 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 295163 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 295165 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@6eb3e086[collection1_shard1_replica2] main]
   [junit4]   2> 295167 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 295167 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 295168 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 295168 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 295168 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 295169 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559180585892577280
   [junit4]   2> 295169 INFO  (searcherExecutor-535-thread-1-processing-n:127.0.0.1:54240_solr x:collection1_shard1_replica2 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2] Registered new searcher Searcher@6eb3e086[collection1_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 295179 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 295179 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 295180 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 295181 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@2d83e285[collection1_shard1_replica2] main]
   [junit4]   2> 295181 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@412685b
   [junit4]   2> 295181 INFO  (searcherExecutor-535-thread-1-processing-n:127.0.0.1:54240_solr x:collection1_shard1_replica2 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [collection1_shard1_replica2] Registered new searcher Searcher@2d83e285[collection1_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 295182 INFO  (Thread-327) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1.shard1.replica2
   [junit4]   2> 295182 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@66775be9[collection1_shard1_replica1] main]
   [junit4]   2> 295184 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 295185 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 295185 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 295186 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 295186 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 295186 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559180585910403072
   [junit4]   2> 295186 INFO  (searcherExecutor-534-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@66775be9[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 295195 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 295206 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 295206 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@32e0e5cd[collection1_shard1_replica1] main]
   [junit4]   2> 295207 INFO  (searcherExecutor-534-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@32e0e5cd[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 295207 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@481b85c9
   [junit4]   2> 295209 INFO  (qtp1697556344-1725) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler I already have the expected version 1 of config
   [junit4]   2> 295209 INFO  (qtp1697556344-1725) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler isStale false , resourceloader org.apache.solr.cloud.ZkSolrResourceLoader
   [junit4]   2> 295209 INFO  (qtp1697556344-1725) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.S.Request [collection1_shard1_replica1]  webapp=/solr path=/config/znodeVersion params={overlay=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 295210 INFO  (Thread-326) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1.shard1.replica1
   [junit4]   2> 295210 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.IndexSchema [collection1_shard1_replica2] Schema name=minimal
   [junit4]   2> 295212 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler I already have the expected version 1 of config
   [junit4]   2> 295212 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.h.SolrConfigHandler isStale false , resourceloader org.apache.solr.cloud.ZkSolrResourceLoader
   [junit4]   2> 295212 INFO  (qtp667814353-1720) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.S.Request [collection1_shard1_replica2]  webapp=/solr path=/config/znodeVersion params={overlay=1&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 295214 INFO  (qtp1697556344-1726) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Took 220.0ms to set the property overlay to be of version 1 for collection collection1
   [junit4]   2> 295214 INFO  (qtp1697556344-1726) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.S.Request [collection1_shard1_replica1]  webapp=/solr path=/config params={wt=json} status=0 QTime=222
   [junit4]   2> 295214 INFO  (qtp1697556344-1726) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@36a69675
   [junit4]   2> 295214 INFO  (qtp1697556344-1726) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1.shard1.replica1
   [junit4]   2> 295215 INFO  (TEST-TestReqParamsAPI.test-seed#[3CF871AE4C014DE1]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):90
   [junit4]   2> 295216 INFO  (TEST-TestReqParamsAPI.test-seed#[3CF871AE4C014DE1]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 295216 INFO  (TEST-TestReqParamsAPI.test-seed#[3CF871AE4C014DE1]) [    ] o.a.s.c.TestSolrConfigHandler going to send config command. path /config/params?wt=json , payload:  {
   [junit4]   2>   'set' : {'x': {                    'a':'A val',
   [junit4]   2>                     'b': 'B val'}
   [junit4]   2>              }
   [junit4]   2>   }
   [junit4]   2> 295216 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 295219 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.CoreContainer Reloading SolrCore 'collection1_shard1_replica2' using configuration from collection collection1
   [junit4]   2> 295221 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 295227 INFO  (qtp1697556344-1729) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Waiting up to 30 secs for 2 replicas to set the property params to be of version 0 for collection collection1
   [junit4]   2> 295228 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 295229 INFO  (Thread-329) [n:127.0.0.1:54240_solr    ] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica2
   [junit4]   2> 295230 INFO  (Thread-328) [n:127.0.0.1:54239_solr    ] o.a.s.c.SolrCore config update listener called for core collection1_shard1_replica1
   [junit4]   2> 295231 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 295232 INFO  (solrHandlerExecutor-542-thread-1-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 295232 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 295232 INFO  (Thread-329) [n:127.0.0.1:54240_solr    ] o.a.s.c.RequestParams conf resource params.json loaded . version : 0 
   [junit4]   2> 295232 INFO  (solrHandlerExecutor-542-thread-2-processing-n:127.0.0.1:54239_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:54239_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.SolrConfigHandler Time elapsed : 0 secs, maxWait 30
   [junit4]   2> 295233 INFO  (Thread-329) [n:127.0.0.1:54240_solr    ] o.a.s.c.RequestParams request params refreshed to version 0
   [junit4]   2> 295233 INFO  (Thread-328) [n:127.0.0.1:54239_solr    ] o.a.s.c.RequestParams conf resource params.json loaded . version : 0 
   [junit4]   2> 295233 INFO  (Thread-328) [n:127.0.0.1:54239_solr    ] o.a.s.c.RequestParams request params refreshed to version 0
   [junit4]   2> 295240 INFO  (SolrConfigHandler-refreshconf) [n:127.0.0.1:54240_solr c:collection1 s:shard1 r:core_node2 x:collection1_shard1_replica2] o.a.s.c.SolrCore [[collection1_shard1_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI_3CF871AE4C014DE1-001/tempDir-001/node2/collection1_shard1_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.ha

[...truncated too long message...]

.0.0.1:60718_m_c%2Fbi) [n:127.0.0.1:60718_m_c%2Fbi c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@24830bc8
   [junit4]   2> 2595571 INFO  (zkCallback-3506-thread-1-processing-n:127.0.0.1:60718_m_c%2Fbi) [n:127.0.0.1:60718_m_c%2Fbi c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 2595572 INFO  (TEST-MultiThreadedOCPTest.test-seed#[3CF871AE4C014DE1]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 2595572 INFO  (TEST-MultiThreadedOCPTest.test-seed#[3CF871AE4C014DE1]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@7527118f{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2595573 INFO  (TEST-MultiThreadedOCPTest.test-seed#[3CF871AE4C014DE1]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5662564e{/m_c/bi,null,UNAVAILABLE}
   [junit4]   2> 2595574 INFO  (TEST-MultiThreadedOCPTest.test-seed#[3CF871AE4C014DE1]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60692 60692
   [junit4]   2> 2595598 INFO  (Thread-3906) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60692 60692
   [junit4]   2> 2602003 WARN  (Thread-3906) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	5	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/collections/ocptest_shardsplit2/state.json
   [junit4]   2> 	4	/solr/collections/ocptest1/state.json
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 	4	/solr/collections/ocptest3/state.json
   [junit4]   2> 	4	/solr/collections/ocptest2/state.json
   [junit4]   2> 	4	/solr/collections/ocptest4/state.json
   [junit4]   2> 	4	/solr/collections/ocptest_shardsplit/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	77	/solr/overseer/queue
   [junit4]   2> 	23	/solr/overseer/queue-work
   [junit4]   2> 	19	/solr/overseer/collection-queue-work
   [junit4]   2> 	6	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=MultiThreadedOCPTest -Dtests.method=test -Dtests.seed=3CF871AE4C014DE1 -Dtests.slow=true -Dtests.locale=en-PH -Dtests.timezone=Antarctica/DumontDUrville -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 46.5s J1 | MultiThreadedOCPTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3CF871AE4C014DE1:B4AC4E74E2FD2019]:0)
   [junit4]    > 	at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:250)
   [junit4]    > 	at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2602137 INFO  (SUITE-MultiThreadedOCPTest-seed#[3CF871AE4C014DE1]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_3CF871AE4C014DE1-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=1024, maxMBSortInHeap=5.6140785014371115, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=en-PH, timezone=Antarctica/DumontDUrville
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=160295360,total=512229376
   [junit4]   2> NOTE: All tests run in this JVM: [TestCloudPseudoReturnFields, RulesTest, SolrInfoMBeanTest, TestStandardQParsers, TestJoin, TestQuerySenderListener, BadIndexSchemaTest, SecurityConfHandlerTest, TestTolerantUpdateProcessorRandomCloud, TolerantUpdateProcessorTest, TestNRTOpen, TestConfigOverlay, TestValueSourceCache, SpatialRPTFieldTypeTest, TestDocSet, TestReversedWildcardFilterFactory, TestDynamicLoading, OverseerStatusTest, CollectionStateFormat2Test, SortSpecParsingTest, TestClusterProperties, ScriptEngineTest, TestFuzzyAnalyzedSuggestions, TestCoreAdminApis, PKIAuthenticationIntegrationTest, JsonLoaderTest, TermVectorComponentTest, QueryResultKeyTest, TestQueryWrapperFilter, BJQParserTest, TestSolrCloudWithDelegationTokens, CoreSorterTest, RegexBoostProcessorTest, TestLMDirichletSimilarityFactory, UpdateRequestProcessorFactoryTest, RemoteQueryErrorTest, DeleteReplicaTest, TestFilteredDocIdSet, TestSolrDeletionPolicy2, ChangedSchemaMergeTest, LoggingHandlerTest, TestMiniSolrCloudCluster, TestQueryTypes, TestStressLiveNodes, AssignTest, TestQueryUtils, BinaryUpdateRequestHandlerTest, ExternalFileFieldSortTest, TestObjectReleaseTracker, AnalyticsQueryTest, HdfsBasicDistributedZk2Test, IndexSchemaTest, DebugComponentTest, FullSolrCloudDistribCmdsTest, BaseCdcrDistributedZkTest, BasicZkTest, SignatureUpdateProcessorFactoryTest, SuggestComponentTest, TestBlendedInfixSuggestions, SpellPossibilityIteratorTest, HdfsRestartWhileUpdatingTest, TestSimpleTrackingShardHandler, TestReload, VersionInfoTest, TestNoOpRegenerator, LeaderElectionIntegrationTest, TestFieldCache, RequestLoggingTest, TestDeleteCollectionOnDownNodes, SmileWriterTest, TestOmitPositions, SuggesterWFSTTest, DistributedFacetPivotLargeTest, TestPathTrie, MigrateRouteKeyTest, SolrGraphiteReporterTest, TestScoreJoinQPScore, TestRebalanceLeaders, TestSlowCompositeReaderWrapper, TestMacros, TemplateUpdateProcessorTest, SolrIndexConfigTest, DocValuesNotIndexedTest, ZkSolrClientTest, TestJsonFacets, RequestHandlersTest, TestDistributedMissingSort, CurrencyFieldOpenExchangeTest, TestZkChroot, CopyFieldTest, RAMDirectoryFactoryTest, CloneFieldUpdateProcessorFactoryTest, TestRealTimeGet, TestImpersonationWithHadoopAuth, TestHashQParserPlugin, ActionThrottleTest, OverriddenZkACLAndCredentialsProvidersTest, PreAnalyzedUpdateProcessorTest, TestJavabinTupleStreamParser, CloudExitableDirectoryReaderTest, HLLSerializationTest, BigEndianAscendingWordSerializerTest, TriLevelCompositeIdRoutingTest, TestFiltering, BasicDistributedZk2Test, TestCloudSchemaless, SolrSlf4jReporterTest, SearchHandlerTest, SortByFunctionTest, ConcurrentDeleteAndCreateCollectionTest, BadCopyFieldTest, TestNonDefinedSimilarityFactory, HdfsRecoverLeaseTest, DistributedExpandComponentTest, TestShardHandlerFactory, EchoParamsTest, AtomicUpdatesTest, BlockCacheTest, TestRequestForwarding, TestXmlQParserPlugin, TestRestoreCore, DefaultValueUpdateProcessorTest, SpellCheckCollatorTest, TestBulkSchemaAPI, TestFieldTypeCollectionResource, TestConfigReload, DataDrivenBlockJoinTest, TestDownShardTolerantSearch, TestFoldingMultitermQuery, SolrMetricManagerTest, FieldMutatingUpdateProcessorTest, DateMathParserTest, TestSchemalessBufferedUpdates, TestUseDocValuesAsStored2, TestMiniSolrCloudClusterSSL, TestDistributedGrouping, PreAnalyzedFieldTest, TestCollectionAPIs, TestCopyFieldCollectionResource, TestSearcherReuse, TestLuceneMatchVersion, TestSubQueryTransformerDistrib, SolrIndexMetricsTest, TestManagedSchema, ShowFileRequestHandlerTest, TestLMJelinekMercerSimilarityFactory, DistribDocExpirationUpdateProcessorTest, ExplicitHLLTest, TestSolrCoreParser, TestBadConfig, TestUninvertingReader, TestDefaultStatsCache, HdfsUnloadDistributedZkTest, PeerSyncReplicationTest, OpenCloseCoreStressTest, TestSQLHandlerNonCloud, HighlighterConfigTest, ClusterStateTest, DistanceUnitsTest, ReturnFieldsTest, BlockJoinFacetRandomTest, TestConfigSetsAPIZkFailure, TestMergePolicyConfig, GraphQueryTest, FieldAnalysisRequestHandlerTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, RecoveryZkTest, TestReplicationHandler, TestRandomFaceting, TestRandomDVFaceting, ZkCLITest, ShardRoutingCustomTest, AutoCommitTest, TestFaceting, TestRecovery, TermVectorComponentDistributedTest, ZkControllerTest, TestStressReorder, TestStressVersions, HardAutoCommitTest, TestRangeQuery, StatsComponentTest, SolrCmdDistributorTest, DirectUpdateHandlerTest, HighlighterTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, CoreAdminHandlerTest, SuggesterTSTTest, SuggesterTest, TestStressLucene, TestTrie, SpatialFilterTest, TestCSVLoader, PolyFieldTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, DistanceFunctionTest, TestSolrDeletionPolicy1, IndexBasedSpellCheckerTest, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, TermsComponentTest, TestIndexingPerformance, RequiredFieldsTest, FastVectorHighlighterTest, TestJmxIntegration, QueryParsingTest, TestPHPSerializedResponseWriter, TestLFUCache, TestPerFieldSimilarity, TestFastWriter, ResourceLoaderTest, URLClassifyProcessorTest, ChaosMonkeyNothingIsSafeTest, CdcrReplicationDistributedZkTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, CustomCollectionTest, DeleteShardTest, DistribCursorPagingTest, HttpPartitionTest, LeaderFailureAfterFreshStartTest, MultiThreadedOCPTest]
   [junit4] Completed [499/691 (2!)] on J1 in 46.53s, 1 test, 1 failure <<< FAILURES!

[...truncated 63278 lines...]


[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 692 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/692/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

All tests passed

Build Log:
[...truncated 75397 lines...]
-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [echo] Checking for malformed docs...
     [exec] 
     [exec] /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/docs/solr-solrj/overview-summary.html
     [exec]   missing: org.apache.solr.client.solrj.io.eval
     [exec] 
     [exec] Missing javadocs were found!

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:775: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:654: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:670: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/common-build.xml:2490: exec returned: 1

Total time: 119 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 691 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/691/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

6 tests failed.
FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration

Error Message:
Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: invalid API spec: apispec/core.SchemaEdit.json

Stack Trace:
org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: invalid API spec: apispec/core.SchemaEdit.json
	at __randomizedtesting.SeedInfo.seed([4F724D419C540B6B:F1F92BEEE52E055E]:0)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:874)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:807)
	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:156)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:919)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:840)
	... 41 more
Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:943)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:807)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:904)
	... 42 more
Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
	at org.apache.solr.api.ApiBag.lambda$getSpec$0(ApiBag.java:229)
	at org.apache.solr.api.Api.getSpec(Api.java:64)
	at org.apache.solr.api.ApiBag.register(ApiBag.java:72)
	at org.apache.solr.core.PluginBag.put(PluginBag.java:215)
	at org.apache.solr.core.PluginBag.init(PluginBag.java:272)
	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:902)
	... 44 more


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Could not find collection:collection2

Stack Trace:
java.lang.AssertionError: Could not find collection:collection2
	at __randomizedtesting.SeedInfo.seed([4F724D419C540B6B:C726729B32A86693]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:159)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:620)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderElectionTest.testElection

Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO

Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at __randomizedtesting.SeedInfo.seed([4F724D419C540B6B:277AF971B468BC1D]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:170)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:106)
	at org.apache.solr.cloud.LeaderElectionTest$ElectorSetup.<init>(LeaderElectionTest.java:114)
	at org.apache.solr.cloud.LeaderElectionTest$ClientThread.<init>(LeaderElectionTest.java:151)
	at org.apache.solr.cloud.LeaderElectionTest$ClientThread.<init>(LeaderElectionTest.java:138)
	at org.apache.solr.cloud.LeaderElectionTest.testElection(LeaderElectionTest.java:286)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1782)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
	at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
	at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
	at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:145)
	... 45 more
Caused by: java.io.IOException: Too many open files
	at sun.nio.ch.IOUtil.makePipe(Native Method)
	at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
	at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
	at java.nio.channels.Selector.open(Selector.java:227)
	at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
	at sun.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at java.lang.Class.newInstance(Class.java:442)
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1779)
	... 51 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionTest

Error Message:
26 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionTest:     1) Thread[id=12301, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    2) Thread[id=12270, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    3) Thread[id=12313, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    4) Thread[id=12273, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)         at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1185)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1110)    5) Thread[id=12310, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    6) Thread[id=12278, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    7) Thread[id=12302, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    8) Thread[id=12294, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    9) Thread[id=12297, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   10) Thread[id=12309, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   11) Thread[id=12277, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   12) Thread[id=12305, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   13) Thread[id=12265, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   14) Thread[id=12274, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   15) Thread[id=12289, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   16) Thread[id=12306, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   17) Thread[id=12281, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   18) Thread[id=12290, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   19) Thread[id=12286, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   20) Thread[id=12269, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   21) Thread[id=12285, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   22) Thread[id=12282, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   23) Thread[id=12293, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   24) Thread[id=12266, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   25) Thread[id=12314, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   26) Thread[id=12298, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 26 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionTest: 
   1) Thread[id=12301, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   2) Thread[id=12270, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   3) Thread[id=12313, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   4) Thread[id=12273, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)
        at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1185)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1110)
   5) Thread[id=12310, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   6) Thread[id=12278, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   7) Thread[id=12302, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   8) Thread[id=12294, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   9) Thread[id=12297, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  10) Thread[id=12309, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  11) Thread[id=12277, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  12) Thread[id=12305, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  13) Thread[id=12265, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  14) Thread[id=12274, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  15) Thread[id=12289, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  16) Thread[id=12306, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  17) Thread[id=12281, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  18) Thread[id=12290, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  19) Thread[id=12286, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  20) Thread[id=12269, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  21) Thread[id=12285, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  22) Thread[id=12282, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  23) Thread[id=12293, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  24) Thread[id=12266, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  25) Thread[id=12314, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  26) Thread[id=12298, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
	at __randomizedtesting.SeedInfo.seed([4F724D419C540B6B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=12301, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    2) Thread[id=12305, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    3) Thread[id=12265, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    4) Thread[id=12313, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    5) Thread[id=12289, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    6) Thread[id=12281, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    7) Thread[id=12273, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    8) Thread[id=12269, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    9) Thread[id=12285, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   10) Thread[id=12293, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   11) Thread[id=12297, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   12) Thread[id=12309, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   13) Thread[id=12277, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=12301, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   2) Thread[id=12305, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   3) Thread[id=12265, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   4) Thread[id=12313, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   5) Thread[id=12289, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   6) Thread[id=12281, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   7) Thread[id=12273, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   8) Thread[id=12269, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   9) Thread[id=12285, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  10) Thread[id=12293, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  11) Thread[id=12297, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  12) Thread[id=12309, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  13) Thread[id=12277, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
	at __randomizedtesting.SeedInfo.seed([4F724D419C540B6B]:0)


FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:
Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>

Stack Trace:
java.lang.AssertionError: Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
	at __randomizedtesting.SeedInfo.seed([4F724D419C540B6B:C726729B32A86693]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotSame(Assert.java:641)
	at org.junit.Assert.assertSame(Assert.java:580)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:250)
	at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11856 lines...]
   [junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/init-core-data-001
   [junit4]   2> 1241938 INFO  (SUITE-MultiThreadedOCPTest-seed#[4F724D419C540B6B]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1241938 INFO  (SUITE-MultiThreadedOCPTest-seed#[4F724D419C540B6B]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 1241946 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1241946 INFO  (Thread-2418) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1241946 INFO  (Thread-2418) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1242052 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.ZkTestServer start zk server on port:63108
   [junit4]   2> 1242144 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1242183 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1242186 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1242189 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1242193 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1242196 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1242199 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1242202 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1242206 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1242209 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1242213 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1245427 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/control-001/cores/collection1
   [junit4]   2> 1245431 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1245434 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@9ef3586{/,null,AVAILABLE}
   [junit4]   2> 1245436 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@20c00ebc{HTTP/1.1,[http/1.1]}{127.0.0.1:63111}
   [junit4]   2> 1245436 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.e.j.s.Server Started @1251835ms
   [junit4]   2> 1245436 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/tempDir-001/control/data, hostContext=/, hostPort=63111, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/control-001/cores}
   [junit4]   2> 1245437 ERROR (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1245437 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 1245437 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1245437 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1245437 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-12T01:58:30.859Z
   [junit4]   2> 1245443 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1245443 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/control-001/solr.xml
   [junit4]   2> 1245456 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1245458 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63108/solr
   [junit4]   2> 1245504 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [n:127.0.0.1:63111_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:63111_
   [junit4]   2> 1245506 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [n:127.0.0.1:63111_    ] o.a.s.c.Overseer Overseer (id=97443165463314436-127.0.0.1:63111_-n_0000000000) starting
   [junit4]   2> 1245520 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [n:127.0.0.1:63111_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63111_
   [junit4]   2> 1245524 INFO  (zkCallback-1547-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1245637 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [n:127.0.0.1:63111_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/control-001/cores
   [junit4]   2> 1245638 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [n:127.0.0.1:63111_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1245642 INFO  (OverseerStateUpdate-97443165463314436-127.0.0.1:63111_-n_0000000000) [n:127.0.0.1:63111_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 1246660 WARN  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1246661 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 1246681 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1246755 WARN  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1246757 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1246776 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1246776 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/control-001/cores/collection1/data/]
   [junit4]   2> 1246777 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@59eaad52
   [junit4]   2> 1246779 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=20.5478515625, floorSegmentMB=1.6240234375, forceMergeDeletesPctAllowed=20.205049605539013, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3834342656412677
   [junit4]   2> 1246784 WARN  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1246874 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1246874 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1246876 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1246876 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1246877 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1729712950, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1246878 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@30e889cc[collection1] main]
   [junit4]   2> 1246880 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1246881 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1246881 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1246882 INFO  (coreLoadExecutor-4717-thread-1-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559090652568879104
   [junit4]   2> 1246883 INFO  (searcherExecutor-4718-thread-1-processing-n:127.0.0.1:63111_ x:collection1 c:control_collection) [n:127.0.0.1:63111_ c:control_collection   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@30e889cc[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1246894 INFO  (coreZkRegister-4710-thread-1-processing-n:127.0.0.1:63111_ x:collection1 c:control_collection) [n:127.0.0.1:63111_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1246894 INFO  (coreZkRegister-4710-thread-1-processing-n:127.0.0.1:63111_ x:collection1 c:control_collection) [n:127.0.0.1:63111_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1246894 INFO  (coreZkRegister-4710-thread-1-processing-n:127.0.0.1:63111_ x:collection1 c:control_collection) [n:127.0.0.1:63111_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:63111/collection1/
   [junit4]   2> 1246894 INFO  (coreZkRegister-4710-thread-1-processing-n:127.0.0.1:63111_ x:collection1 c:control_collection) [n:127.0.0.1:63111_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1246894 INFO  (coreZkRegister-4710-thread-1-processing-n:127.0.0.1:63111_ x:collection1 c:control_collection) [n:127.0.0.1:63111_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:63111/collection1/ has no replicas
   [junit4]   2> 1246902 INFO  (coreZkRegister-4710-thread-1-processing-n:127.0.0.1:63111_ x:collection1 c:control_collection) [n:127.0.0.1:63111_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:63111/collection1/ shard1
   [junit4]   2> 1247056 INFO  (coreZkRegister-4710-thread-1-processing-n:127.0.0.1:63111_ x:collection1 c:control_collection) [n:127.0.0.1:63111_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1247207 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1247212 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:63108/solr ready
   [junit4]   2> 1247212 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1247212 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
   [junit4]   2> 1247630 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/shard-1-001/cores/collection1
   [junit4]   2> 1247631 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/shard-1-001
   [junit4]   2> 1247632 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1247634 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7edc9413{/,null,AVAILABLE}
   [junit4]   2> 1247634 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@533586c9{HTTP/1.1,[http/1.1]}{127.0.0.1:63117}
   [junit4]   2> 1247634 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.e.j.s.Server Started @1254034ms
   [junit4]   2> 1247634 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=63117, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/shard-1-001/cores}
   [junit4]   2> 1247634 ERROR (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1247634 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 1247635 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1247635 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1247635 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-12T01:58:33.057Z
   [junit4]   2> 1247639 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1247639 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/shard-1-001/solr.xml
   [junit4]   2> 1247730 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1247730 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63108/solr
   [junit4]   2> 1247747 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [n:127.0.0.1:63117_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1247756 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [n:127.0.0.1:63117_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63117_
   [junit4]   2> 1247761 INFO  (zkCallback-1547-thread-2-processing-n:127.0.0.1:63111_) [n:127.0.0.1:63111_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1247762 INFO  (zkCallback-1551-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1247762 INFO  (zkCallback-1557-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1247837 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [n:127.0.0.1:63117_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/shard-1-001/cores
   [junit4]   2> 1247837 INFO  (TEST-MultiThreadedOCPTest.test-seed#[4F724D419C540B6B]) [n:127.0.0.1:63117_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1247841 INFO  (OverseerStateUpdate-97443165463314436-127.0.0.1:63111_-n_0000000000) [n:127.0.0.1:63111_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 1247947 INFO  (zkCallback-1557-thread-2-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 1248856 WARN  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1248856 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 1248867 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1248934 WARN  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1248936 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1248955 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1248955 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest_4F724D419C540B6B-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 1248956 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@59eaad52
   [junit4]   2> 1248957 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=20.5478515625, floorSegmentMB=1.6240234375, forceMergeDeletesPctAllowed=20.205049605539013, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3834342656412677
   [junit4]   2> 1248963 WARN  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1249029 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1249029 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1249031 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1249031 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1249032 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1729712950, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1249033 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@746680f1[collection1] main]
   [junit4]   2> 1249035 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1249035 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1249035 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1249036 INFO  (searcherExecutor-4729-thread-1-processing-n:127.0.0.1:63117_ x:collection1 c:collection1) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@746680f1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1249037 INFO  (coreLoadExecutor-4728-thread-1-processing-n:127.0.0.1:63117_) [n:127.0.0.1:63117_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559090654828560384
   [junit4]   2> 1249047 INFO  (coreZkRegister-4723-thread-1-processing-n:127.0.0.1:63117_ x:collection1 c:collection1) [n:127.0.0.1:63117_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1249047 INFO  (coreZkRegister-4723-thread-1-processing-n:127.0.0.1:63117_ x:collection1 c:collection1) [n:127.0.0.1:63117_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1249048 INFO  (

[...truncated too long message...]

D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   [junit4]    >   25) Thread[id=12314, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   [junit4]    >   26) Thread[id=12298, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-EventThread, state=WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4F724D419C540B6B]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=12301, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >    2) Thread[id=12305, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   [junit4]    >    3) Thread[id=12265, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   [junit4]    >    4) Thread[id=12313, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >    5) Thread[id=12289, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >    6) Thread[id=12281, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >    7) Thread[id=12273, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >    8) Thread[id=12269, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   [junit4]    >    9) Thread[id=12285, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   [junit4]    >   10) Thread[id=12293, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   [junit4]    >   11) Thread[id=12297, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >   12) Thread[id=12309, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >   13) Thread[id=12277, name=TEST-LeaderElectionTest.testElection-seed#[4F724D419C540B6B]-SendThread(127.0.0.1:64939), state=TIMED_WAITING, group=TGRP-LeaderElectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4F724D419C540B6B]:0)
   [junit4] Completed [363/691 (4!)] on J1 in 84.01s, 5 tests, 3 errors <<< FAILURES!

[...truncated 63328 lines...]
-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [echo] Checking for malformed docs...
     [exec] 
     [exec] /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/docs/solr-solrj/overview-summary.html
     [exec]   missing: org.apache.solr.client.solrj.io.eval
     [exec] 
     [exec] Missing javadocs were found!

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:775: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:654: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:670: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/common-build.xml:2490: exec returned: 1

Total time: 89 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 690 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/690/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.lucene.codecs.simpletext.TestSimpleTextPointsFormat.testWithExceptions

Error Message:
this writer hit an unrecoverable error; cannot complete forceMerge

Stack Trace:
java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot complete forceMerge
	at __randomizedtesting.SeedInfo.seed([CCE1E867577CFFF6:6EB2741BD8F2B00C]:0)
	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1931)
	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1881)
	at org.apache.lucene.index.RandomIndexWriter.forceMerge(RandomIndexWriter.java:429)
	at org.apache.lucene.index.BasePointsFormatTestCase.verify(BasePointsFormatTestCase.java:701)
	at org.apache.lucene.index.BasePointsFormatTestCase.testWithExceptions(BasePointsFormatTestCase.java:224)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.lucene.index.CorruptIndexException: Problem reading index from MockDirectoryWrapper(NIOFSDirectory@/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/build/codecs/test/J0/temp/lucene.codecs.simpletext.TestSimpleTextPointsFormat_CCE1E867577CFFF6-001/tempDir-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4d6de658) (resource=MockDirectoryWrapper(NIOFSDirectory@/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/build/codecs/test/J0/temp/lucene.codecs.simpletext.TestSimpleTextPointsFormat_CCE1E867577CFFF6-001/tempDir-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4d6de658))
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:140)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
	at org.apache.lucene.index.ReadersAndUpdates.getReaderForMerge(ReadersAndUpdates.java:617)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4293)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3940)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)
Caused by: java.io.FileNotFoundException: a random IOException (_0.inf)
	at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowIOExceptionOnOpen(MockDirectoryWrapper.java:575)
	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:744)
	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:137)
	at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1072)
	at org.apache.lucene.codecs.simpletext.SimpleTextFieldInfosFormat.read(SimpleTextFieldInfosFormat.java:73)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:107)
	... 7 more


FAILED:  org.apache.solr.cloud.TestCloudPivotFacet.test

Error Message:
pivot_z_s1,pivot_ti,pivot_dt: count query failed: {main(rows=0&q=id:[*+TO+360]&fq=id:[*+TO+202]&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_z_s1&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_d1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_tl),extra(fq={!term+f%3Dpivot_z_s1}g)}: No live SolrServers available to handle this request:[http://127.0.0.1:60349/collection1, http://127.0.0.1:60323/collection1]

Stack Trace:
java.lang.RuntimeException: pivot_z_s1,pivot_ti,pivot_dt: count query failed: {main(rows=0&q=id:[*+TO+360]&fq=id:[*+TO+202]&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_z_s1&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_d1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_tl),extra(fq={!term+f%3Dpivot_z_s1}g)}: No live SolrServers available to handle this request:[http://127.0.0.1:60349/collection1, http://127.0.0.1:60323/collection1]
	at __randomizedtesting.SeedInfo.seed([8F2887FB7572E232:77CB821DB8E8FCA]:0)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:320)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:285)
	at org.apache.solr.cloud.TestCloudPivotFacet.test(TestCloudPivotFacet.java:242)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:60349/collection1, http://127.0.0.1:60323/collection1]
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:414)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1358)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1109)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1042)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotData(TestCloudPivotFacet.java:346)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:316)
	... 43 more
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60323/collection1: no servers hosting shard: shard2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:610)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:435)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:387)
	... 51 more




Build Log:
[...truncated 6209 lines...]
   [junit4] Suite: org.apache.lucene.codecs.simpletext.TestSimpleTextPointsFormat
   [junit4] IGNOR/A 0.02s J0 | TestSimpleTextPointsFormat.testRandomBinaryBig
   [junit4]    > Assumption #1: 'nightly' test group is disabled (@Nightly())
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSimpleTextPointsFormat -Dtests.method=testWithExceptions -Dtests.seed=CCE1E867577CFFF6 -Dtests.slow=true -Dtests.locale=uk-UA -Dtests.timezone=Asia/Qatar -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.93s J0 | TestSimpleTextPointsFormat.testWithExceptions <<<
   [junit4]    > Throwable #1: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot complete forceMerge
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CCE1E867577CFFF6:6EB2741BD8F2B00C]:0)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1931)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1881)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.forceMerge(RandomIndexWriter.java:429)
   [junit4]    > 	at org.apache.lucene.index.BasePointsFormatTestCase.verify(BasePointsFormatTestCase.java:701)
   [junit4]    > 	at org.apache.lucene.index.BasePointsFormatTestCase.testWithExceptions(BasePointsFormatTestCase.java:224)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.lucene.index.CorruptIndexException: Problem reading index from MockDirectoryWrapper(NIOFSDirectory@/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/build/codecs/test/J0/temp/lucene.codecs.simpletext.TestSimpleTextPointsFormat_CCE1E867577CFFF6-001/tempDir-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4d6de658) (resource=MockDirectoryWrapper(NIOFSDirectory@/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/build/codecs/test/J0/temp/lucene.codecs.simpletext.TestSimpleTextPointsFormat_CCE1E867577CFFF6-001/tempDir-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4d6de658))
   [junit4]    > 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:140)
   [junit4]    > 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReaderForMerge(ReadersAndUpdates.java:617)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4293)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3940)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)
   [junit4]    > Caused by: java.io.FileNotFoundException: a random IOException (_0.inf)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowIOExceptionOnOpen(MockDirectoryWrapper.java:575)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:744)
   [junit4]    > 	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:137)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.openChecksumInput(MockDirectoryWrapper.java:1072)
   [junit4]    > 	at org.apache.lucene.codecs.simpletext.SimpleTextFieldInfosFormat.read(SimpleTextFieldInfosFormat.java:73)
   [junit4]    > 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:107)
   [junit4]    > 	... 7 more
   [junit4] IGNOR/A 0.01s J0 | TestSimpleTextPointsFormat.testMergeStability
   [junit4]    > Assumption #1: merge is not stable
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/build/codecs/test/J0/temp/lucene.codecs.simpletext.TestSimpleTextPointsFormat_CCE1E867577CFFF6-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=1626, maxMBSortInHeap=5.745076713359786, sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=uk-UA, timezone=Asia/Qatar
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=153242552,total=251133952
   [junit4]   2> NOTE: All tests run in this JVM: [TestSimpleTextFieldInfoFormat, TestDirectPostingsFormat, TestSimpleTextTermVectorsFormat, TestFSTOrdPostingsFormat, TestSimpleTextPointsFormat]
   [junit4] Completed [16/20 (1!)] on J0 in 22.49s, 18 tests, 1 error, 2 skipped <<< FAILURES!

[...truncated 6267 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestCloudPivotFacet
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/init-core-data-001
   [junit4]   2> 2783620 INFO  (SUITE-TestCloudPivotFacet-seed#[8F2887FB7572E232]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2783621 INFO  (SUITE-TestCloudPivotFacet-seed#[8F2887FB7572E232]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 2783621 INFO  (SUITE-TestCloudPivotFacet-seed#[8F2887FB7572E232]-worker) [    ] o.a.s.c.TestCloudPivotFacet init'ing useFieldRandomizedFactor = 5
   [junit4]   2> 2783625 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2783625 INFO  (Thread-3981) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2783625 INFO  (Thread-3981) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2783732 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.ZkTestServer start zk server on port:60314
   [junit4]   2> 2783762 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15a2d65ad9f0000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2783779 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2783789 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2783796 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2783801 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2783806 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2783811 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2783815 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2783822 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2783828 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2783838 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2783852 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2785808 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/control-001/cores/collection1
   [junit4]   2> 2785810 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2785812 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3da2d79e{/,null,AVAILABLE}
   [junit4]   2> 2785813 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5c0943c8{HTTP/1.1,[http/1.1]}{127.0.0.1:60317}
   [junit4]   2> 2785813 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server Started @2790932ms
   [junit4]   2> 2785813 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/tempDir-001/control/data, hostContext=/, hostPort=60317, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/control-001/cores}
   [junit4]   2> 2785813 ERROR (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2785814 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2785814 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2785814 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2785814 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-11T13:38:44.903Z
   [junit4]   2> 2785820 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2785820 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/control-001/solr.xml
   [junit4]   2> 2785829 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2785829 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60314/solr
   [junit4]   2> 2785884 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60317_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60317_
   [junit4]   2> 2785886 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60317_    ] o.a.s.c.Overseer Overseer (id=97440256669712388-127.0.0.1:60317_-n_0000000000) starting
   [junit4]   2> 2785902 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60317_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60317_
   [junit4]   2> 2785906 INFO  (zkCallback-2726-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2785982 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60317_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/control-001/cores
   [junit4]   2> 2785982 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60317_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2785988 INFO  (OverseerStateUpdate-97440256669712388-127.0.0.1:60317_-n_0000000000) [n:127.0.0.1:60317_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 2787017 WARN  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2787017 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2787036 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2787133 WARN  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2787135 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2787161 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 2787161 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/control-001/cores/collection1/data/]
   [junit4]   2> 2787161 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ab9ee
   [junit4]   2> 2787162 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 2787175 WARN  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2787251 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2787251 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2787252 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2787252 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2787252 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=469903837, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 2787253 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a3aeace[collection1] main]
   [junit4]   2> 2787255 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2787255 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2787256 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2787257 INFO  (searcherExecutor-9122-thread-1-processing-n:127.0.0.1:60317_ x:collection1 c:control_collection) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6a3aeace[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2787257 INFO  (coreLoadExecutor-9121-thread-1-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559044110518583296
   [junit4]   2> 2787270 INFO  (coreZkRegister-9114-thread-1-processing-n:127.0.0.1:60317_ x:collection1 c:control_collection) [n:127.0.0.1:60317_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2787270 INFO  (coreZkRegister-9114-thread-1-processing-n:127.0.0.1:60317_ x:collection1 c:control_collection) [n:127.0.0.1:60317_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2787270 INFO  (coreZkRegister-9114-thread-1-processing-n:127.0.0.1:60317_ x:collection1 c:control_collection) [n:127.0.0.1:60317_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60317/collection1/
   [junit4]   2> 2787270 INFO  (coreZkRegister-9114-thread-1-processing-n:127.0.0.1:60317_ x:collection1 c:control_collection) [n:127.0.0.1:60317_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2787271 INFO  (coreZkRegister-9114-thread-1-processing-n:127.0.0.1:60317_ x:collection1 c:control_collection) [n:127.0.0.1:60317_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60317/collection1/ has no replicas
   [junit4]   2> 2787280 INFO  (coreZkRegister-9114-thread-1-processing-n:127.0.0.1:60317_ x:collection1 c:control_collection) [n:127.0.0.1:60317_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60317/collection1/ shard1
   [junit4]   2> 2787439 INFO  (coreZkRegister-9114-thread-1-processing-n:127.0.0.1:60317_ x:collection1 c:control_collection) [n:127.0.0.1:60317_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2787523 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2787526 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60314/solr ready
   [junit4]   2> 2787526 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2787526 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
   [junit4]   2> 2787895 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-1-001/cores/collection1
   [junit4]   2> 2787896 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-1-001
   [junit4]   2> 2787897 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2787899 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@599ae561{/,null,AVAILABLE}
   [junit4]   2> 2787899 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5a50a6b9{HTTP/1.1,[http/1.1]}{127.0.0.1:60323}
   [junit4]   2> 2787900 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server Started @2793018ms
   [junit4]   2> 2787900 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=60323, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-1-001/cores}
   [junit4]   2> 2787900 ERROR (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2787900 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2787900 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2787900 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2787900 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-11T13:38:46.989Z
   [junit4]   2> 2787904 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2787904 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-1-001/solr.xml
   [junit4]   2> 2787929 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2787929 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60314/solr
   [junit4]   2> 2787948 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2787958 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60323_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60323_
   [junit4]   2> 2787961 INFO  (zkCallback-2730-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2787961 INFO  (zkCallback-2726-thread-2-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2787962 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2788038 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60323_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-1-001/cores
   [junit4]   2> 2788038 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60323_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2788045 INFO  (OverseerStateUpdate-97440256669712388-127.0.0.1:60317_-n_0000000000) [n:127.0.0.1:60317_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 2788151 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2789064 WARN  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2789066 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2789078 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2789183 WARN  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2789186 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2789215 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2789215 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 2789215 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ab9ee
   [junit4]   2> 2789217 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 2789228 WARN  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2789306 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2789306 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2789308 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2789308 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2789310 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=469903837, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 2789311 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@16cb76a6[collection1] main]
   [junit4]   2> 2789313 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2789314 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2789315 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2789316 INFO  (searcherExecutor-9133-thread-1-processing-n:127.0.0.1:60323_ x:collection1 c:collection1) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@16cb76a6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2789317 INFO  (coreLoadExecutor-9132-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559044112678649856
   [junit4]   2> 2789330 INFO  (coreZkRegister-9127-thread-1-processing-n:127.0.0.1:60323_ x:collection1 c:collection1) [n:127.0.0.1:60323_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2789330 INFO  (coreZkRegister-9127-thread-1-processing-n:127.0.0.1:60323_ x:collection1 c:collection1) [n:127.0.0.1:60323_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2789330 INFO  (coreZkRegister-9127-thread-1-processing-n:127.0.0.1:60323_ x:collection1 c:collection1) [n:127.0.0.1:60323_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60323/collection1/
   [junit4]   2> 2789330 INFO  (coreZkRegister-9127-thread-1-processing-n:127.0.0.1:60323_ x:collection1 c:collection1) [n:127.0.0.1:60323_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2789330 INFO  (coreZkRegister-9127-thread-1-processing-n:127.0.0.1:60323_ x:collection1 c:collection1) [n:127.0.0.1:60323_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60323/collection1/ has no replicas
   [junit4]   2> 2789339 INFO  (coreZkRegister-9127-thread-1-processing-n:127.0.0.1:60323_ x:collection1 c:collection1) [n:127.0.0.1:60323_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60323/collection1/ shard2
   [junit4]   2> 2789469 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2789515 INFO  (coreZkRegister-9127-thread-1-processing-n:127.0.0.1:60323_ x:collection1 c:collection1) [n:127.0.0.1:60323_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2789632 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2790157 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-2-001/cores/collection1
   [junit4]   2> 2790158 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-2-001
   [junit4]   2> 2790158 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2790160 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@22983f1b{/,null,AVAILABLE}
   [junit4]   2> 2790161 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1cd88b26{HTTP/1.1,[http/1.1]}{127.0.0.1:60327}
   [junit4]   2> 2790161 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server Started @2795279ms
   [junit4]   2> 2790161 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=60327, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-2-001/cores}
   [junit4]   2> 2790161 ERROR (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2790161 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2790161 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2790161 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2790161 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-11T13:38:49.250Z
   [junit4]   2> 2790166 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2790166 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-2-001/solr.xml
   [junit4]   2> 2790250 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2790251 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60314/solr
   [junit4]   2> 2790268 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2790276 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60327_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60327_
   [junit4]   2> 2790279 INFO  (zkCallback-2730-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2790279 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2790280 INFO  (zkCallback-2742-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2790280 INFO  (zkCallback-2726-thread-3-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2790366 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60327_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-2-001/cores
   [junit4]   2> 2790367 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60327_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2790371 INFO  (OverseerStateUpdate-97440256669712388-127.0.0.1:60317_-n_0000000000) [n:127.0.0.1:60317_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 2790478 INFO  (zkCallback-2742-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2790478 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2791386 WARN  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2791386 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2791398 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2791486 WARN  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2791488 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2791504 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2791505 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 2791505 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ab9ee
   [junit4]   2> 2791516 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 2791557 WARN  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2791622 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2791622 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2791623 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2791623 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2791625 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=469903837, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 2791627 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@a002ee0[collection1] main]
   [junit4]   2> 2791628 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2791629 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2791629 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2791630 INFO  (searcherExecutor-9144-thread-1-processing-n:127.0.0.1:60327_ x:collection1 c:collection1) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@a002ee0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2791665 INFO  (coreLoadExecutor-9143-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559044115140706304
   [junit4]   2> 2791677 INFO  (coreZkRegister-9138-thread-1-processing-n:127.0.0.1:60327_ x:collection1 c:collection1) [n:127.0.0.1:60327_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2791677 INFO  (coreZkRegister-9138-thread-1-processing-n:127.0.0.1:60327_ x:collection1 c:collection1) [n:127.0.0.1:60327_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2791677 INFO  (coreZkRegister-9138-thread-1-processing-n:127.0.0.1:60327_ x:collection1 c:collection1) [n:127.0.0.1:60327_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60327/collection1/
   [junit4]   2> 2791677 INFO  (coreZkRegister-9138-thread-1-processing-n:127.0.0.1:60327_ x:collection1 c:collection1) [n:127.0.0.1:60327_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2791677 INFO  (coreZkRegister-9138-thread-1-processing-n:127.0.0.1:60327_ x:collection1 c:collection1) [n:127.0.0.1:60327_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60327/collection1/ has no replicas
   [junit4]   2> 2791683 INFO  (coreZkRegister-9138-thread-1-processing-n:127.0.0.1:60327_ x:collection1 c:collection1) [n:127.0.0.1:60327_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60327/collection1/ shard1
   [junit4]   2> 2791789 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2791789 INFO  (zkCallback-2742-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2791841 INFO  (coreZkRegister-9138-thread-1-processing-n:127.0.0.1:60327_ x:collection1 c:collection1) [n:127.0.0.1:60327_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2791949 INFO  (zkCallback-2742-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2791949 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 2792375 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-3-001/cores/collection1
   [junit4]   2> 2792376 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-3-001
   [junit4]   2> 2792377 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2792379 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@24038cec{/,null,AVAILABLE}
   [junit4]   2> 2792379 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5ac6b97d{HTTP/1.1,[http/1.1]}{127.0.0.1:60338}
   [junit4]   2> 2792379 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server Started @2797498ms
   [junit4]   2> 2792380 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=60338, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-3-001/cores}
   [junit4]   2> 2792380 ERROR (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2792380 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2792380 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2792380 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2792380 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-11T13:38:51.469Z
   [junit4]   2> 2792387 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2792387 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-3-001/solr.xml
   [junit4]   2> 2792398 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2792399 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60314/solr
   [junit4]   2> 2792416 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60338_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 2792426 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60338_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60338_
   [junit4]   2> 2792429 INFO  (zkCallback-2726-thread-3-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2792430 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2792430 INFO  (zkCallback-2742-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2792430 INFO  (zkCallback-2730-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2792432 INFO  (zkCallback-2748-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2792545 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60338_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-3-001/cores
   [junit4]   2> 2792545 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60338_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2792548 INFO  (zkCallback-2742-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2792548 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2792553 INFO  (OverseerStateUpdate-97440256669712388-127.0.0.1:60317_-n_0000000000) [n:127.0.0.1:60317_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 2792660 INFO  (zkCallback-2742-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2792660 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2792660 INFO  (zkCallback-2748-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2793571 WARN  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2793571 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2793586 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2793691 WARN  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2793693 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2793719 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2793719 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-3-001/cores/collection1/data/]
   [junit4]   2> 2793720 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@751ab9ee
   [junit4]   2> 2793723 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 2793734 WARN  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 2793827 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2793827 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2793829 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2793829 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2793830 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=469903837, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 2793831 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@701625ae[collection1] main]
   [junit4]   2> 2793833 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2793834 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2793835 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2793836 INFO  (searcherExecutor-9155-thread-1-processing-n:127.0.0.1:60338_ x:collection1 c:collection1) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@701625ae[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2793838 INFO  (coreLoadExecutor-9154-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1559044117419261952
   [junit4]   2> 2793846 INFO  (coreZkRegister-9149-thread-1-processing-n:127.0.0.1:60338_ x:collection1 c:collection1) [n:127.0.0.1:60338_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 2793847 INFO  (updateExecutor-2745-thread-1-processing-n:127.0.0.1:60338_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:60338_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 2793848 INFO  (recoveryExecutor-2746-thread-1-processing-n:127.0.0.1:60338_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:60338_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 2793848 INFO  (recoveryExecutor-2746-thread-1-processing-n:127.0.0.1:60338_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:60338_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 2793848 INFO  (recoveryExecutor-2746-thread-1-processing-n:127.0.0.1:60338_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:60338_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
   [junit4]   2> 2793848 INFO  (recoveryExecutor-2746-thread-1-processing-n:127.0.0.1:60338_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:60338_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 2793848 INFO  (recoveryExecutor-2746-thread-1-processing-n:127.0.0.1:60338_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:60338_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:60323/collection1/] and I am [http://127.0.0.1:60338/collection1/]
   [junit4]   2> 2793852 INFO  (recoveryExecutor-2746-thread-1-processing-n:127.0.0.1:60338_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:60338_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:60323]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:60338_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 2793859 INFO  (qtp1301382051-16623) [n:127.0.0.1:60323_    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 2793859 INFO  (qtp1301382051-16623) [n:127.0.0.1:60323_    ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard2 of collection1) have state: recovering
   [junit4]   2> 2793860 INFO  (qtp1301382051-16623) [n:127.0.0.1:60323_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60338_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:60338","node_name":"127.0.0.1:60338_","state":"down"}
   [junit4]   2> 2793956 INFO  (zkCallback-2742-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2793956 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2793956 INFO  (zkCallback-2748-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2794863 INFO  (qtp1301382051-16623) [n:127.0.0.1:60323_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60338_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:60338","node_name":"127.0.0.1:60338_","state":"recovering"}
   [junit4]   2> 2794863 INFO  (qtp1301382051-16623) [n:127.0.0.1:60323_    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 2794863 INFO  (qtp1301382051-16623) [n:127.0.0.1:60323_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60338_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1004
   [junit4]   2> 2795011 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-4-001/cores/collection1
   [junit4]   2> 2795014 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-4-001
   [junit4]   2> 2795015 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 2795017 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@26daf00e{/,null,AVAILABLE}
   [junit4]   2> 2795018 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@647ac6d5{HTTP/1.1,[http/1.1]}{127.0.0.1:60349}
   [junit4]   2> 2795018 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.e.j.s.Server Started @2800136ms
   [junit4]   2> 2795018 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/, hostPort=60349, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-4-001/cores}
   [junit4]   2> 2795018 ERROR (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2795019 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.5.0
   [junit4]   2> 2795019 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2795019 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2795019 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-02-11T13:38:54.108Z
   [junit4]   2> 2795025 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2795025 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-4-001/solr.xml
   [junit4]   2> 2795039 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2795040 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60314/solr
   [junit4]   2> 2795057 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60349_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2795066 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60349_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60349_
   [junit4]   2> 2795071 INFO  (zkCallback-2742-thread-1-processing-n:127.0.0.1:60327_) [n:127.0.0.1:60327_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2795071 INFO  (zkCallback-2730-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2795071 INFO  (zkCallback-2726-thread-3-processing-n:127.0.0.1:60317_) [n:127.0.0.1:60317_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2795072 INFO  (zkCallback-2736-thread-1-processing-n:127.0.0.1:60323_) [n:127.0.0.1:60323_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2795072 INFO  (zkCallback-2755-thread-1-processing-n:127.0.0.1:60349_) [n:127.0.0.1:60349_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2795073 INFO  (zkCallback-2748-thread-1-processing-n:127.0.0.1:60338_) [n:127.0.0.1:60338_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 2795173 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60349_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001/shard-4-001/cores
   [junit4]   2> 2795173 INFO  (TEST-TestCloudPivotFacet.test-seed#[8F2887FB7572E232]) [n:127.0.0.1:60349_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2

[...truncated too long message...]

X/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_8F2887FB7572E232-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {pivot_dt1=PostingsFormat(name=LuceneVarGapFixedInterval), dense_pivot_i1=FST50, pivot_l1=PostingsFormat(name=Memory doPackFST= true), range_facet_l_dv=FST50, multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), pivot_f1=PostingsFormat(name=LuceneVarGapFixedInterval), intDefault=PostingsFormat(name=Memory doPackFST= true), pivot_d1=PostingsFormat(name=Memory doPackFST= true), pivot_x_s=PostingsFormat(name=Asserting), pivot_b1=PostingsFormat(name=LuceneVarGapFixedInterval), pivot_dt=FST50, dense_pivot_ti1=FST50, range_facet_l=PostingsFormat(name=Memory doPackFST= true), pivot_x_s1=PostingsFormat(name=Memory doPackFST= true), pivot_y_s=PostingsFormat(name=LuceneVarGapFixedInterval), pivot_z_s=FST50, dense_pivot_ti=PostingsFormat(name=Memory doPackFST= true), pivot_z_s1=PostingsFormat(name=LuceneVarGapFixedInterval), pivot_tl1=PostingsFormat(name=LuceneVarGapFixedInterval), id=FST50, dense_pivot_x_s1=PostingsFormat(name=Memory doPackFST= true), timestamp=PostingsFormat(name=Memory doPackFST= true), dense_pivot_y_s=PostingsFormat(name=LuceneVarGapFixedInterval), pivot_td1=PostingsFormat(name=LuceneVarGapFixedInterval), pivot_tdt=PostingsFormat(name=Asserting), pivot_i1=FST50, pivot_tf1=PostingsFormat(name=Memory doPackFST= true), pivot_td=FST50, pivot_tf=PostingsFormat(name=Asserting), pivot_f=FST50, pivot_ti=PostingsFormat(name=Memory doPackFST= true), _version_=PostingsFormat(name=Memory doPackFST= true), pivot_i=PostingsFormat(name=LuceneVarGapFixedInterval), pivot_tl=FST50, pivot_l=PostingsFormat(name=Asserting), pivot_y_s1=PostingsFormat(name=Asserting), pivot_ti1=PostingsFormat(name=Asserting), range_facet_i_dv=PostingsFormat(name=Memory doPackFST= true), dense_pivot_i=PostingsFormat(name=LuceneVarGapFixedInterval), pivot_b=FST50, dense_pivot_y_s1=PostingsFormat(name=Asserting), pivot_d=PostingsFormat(name=Asserting), pivot_tdt1=PostingsFormat(name=Memory doPackFST= true), dense_pivot_x_s=FST50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene54), range_facet_i_dv=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=302, maxMBSortInHeap=5.8501188030809566, sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=sv-SE, timezone=Pacific/Yap
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=202880432,total=503840768
   [junit4]   2> NOTE: All tests run in this JVM: [QueryParsingTest, DistributedFacetPivotWhiteBoxTest, TestQueryUtils, DistributedFacetPivotLargeTest, TestLeaderElectionZkExpiry, ZkSolrClientTest, TestSuggestSpellingConverter, ResourceLoaderTest, ManagedSchemaRoundRobinCloudTest, TestConfigReload, CircularListTest, SpellingQueryConverterTest, BadCopyFieldTest, SolrMetricManagerTest, TestCrossCoreJoin, RegexBoostProcessorTest, BlockJoinFacetRandomTest, SignatureUpdateProcessorFactoryTest, TestTrackingShardHandlerFactory, ClassificationUpdateProcessorIntegrationTest, OpenCloseCoreStressTest, EchoParamsTest, LeaderInitiatedRecoveryOnCommitTest, TestAuthenticationFramework, TestValueSourceCache, TestSimpleTrackingShardHandler, TestSmileRequest, CustomCollectionTest, ClusterStateTest, TestDefaultStatsCache, DistributedIntervalFacetingTest, ChangedSchemaMergeTest, TestSchemalessBufferedUpdates, BadIndexSchemaTest, TestImpersonationWithHadoopAuth, TestQuerySenderNoQuery, CurrencyFieldOpenExchangeTest, TestTrieFacet, TestIndexingPerformance, StressHdfsTest, PeerSyncReplicationTest, TermVectorComponentTest, SolrInfoMBeanTest, TestMiniSolrCloudCluster, TestPHPSerializedResponseWriter, FieldMutatingUpdateProcessorTest, InfoHandlerTest, TestBulkSchemaAPI, BinaryUpdateRequestHandlerTest, LukeRequestHandlerTest, HdfsRecoverLeaseTest, SliceStateTest, TestFieldCacheWithThreads, SuggesterTSTTest, TestPathTrie, TestSolrCloudWithDelegationTokens, ExternalFileFieldSortTest, DistanceUnitsTest, TestSolrCoreProperties, TestLuceneMatchVersion, FullSolrCloudDistribCmdsTest, TestConfig, TestJmxMonitoredMap, TestRandomFaceting, SuggestComponentTest, MultiThreadedOCPTest, TestNonDefinedSimilarityFactory, RemoteQueryErrorTest, HdfsBasicDistributedZk2Test, BJQParserTest, CdcrRequestHandlerTest, MBeansHandlerTest, TestSolrCloudWithSecureImpersonation, SolrCmdDistributorTest, NotRequiredUniqueKeyTest, DistributedFacetPivotSmallAdvancedTest, TestJoin, ExplicitHLLTest, SpatialHeatmapFacetsTest, CloneFieldUpdateProcessorFactoryTest, TestFiltering, TestFastWriter, SolrGraphiteReporterTest, TestSolrCloudSnapshots, BigEndianAscendingWordSerializerTest, TestCollectionAPIs, HdfsRestartWhileUpdatingTest, TestNoOpRegenerator, TestRangeQuery, TestReplicationHandlerBackup, HLLSerializationTest, FileBasedSpellCheckerTest, AnalysisErrorHandlingTest, TestIntervalFaceting, RulesTest, StatsReloadRaceTest, TestJettySolrRunner, SuggesterWFSTTest, OverriddenZkACLAndCredentialsProvidersTest, SaslZkACLProviderTest, StatsComponentTest, V2ApiIntegrationTest, TestRecoveryHdfs, TestNumericTerms32, TestCustomDocTransformer, TestFreeTextSuggestions, TestSSLRandomization, TestManagedSchemaThreadSafety, TestSolrQueryParserResource, DistribDocExpirationUpdateProcessorTest, MinimalSchemaTest, VersionInfoTest, DeleteReplicaTest, QueryResultKeyTest, DeleteStatusTest, CopyFieldTest, AnalyticsQueryTest, SuggesterTest, TestClusterProperties, TestRequestForwarding, TestFuzzyAnalyzedSuggestions, TestRealTimeGet, BaseCdcrDistributedZkTest, SolrSlf4jReporterTest, DistributedQueryComponentOptimizationTest, DataDrivenBlockJoinTest, TestManagedSchema, TestSolr4Spatial, TermVectorComponentDistributedTest, FieldAnalysisRequestHandlerTest, CollectionStateFormat2Test, TestSerializedLuceneMatchVersion, BasicFunctionalityTest, SpellPossibilityIteratorTest, GraphQueryTest, TestCloudManagedSchema, RequestHandlersTest, TestRecovery, CSVRequestHandlerTest, DateMathParserTest, TestAtomicUpdateErrorCases, ZkStateReaderTest, PreAnalyzedFieldTest, IndexSchemaTest, TestInfoStreamLogging, TolerantUpdateProcessorTest, TestPerFieldSimilarityWithDefaultOverride, TestConfigOverlay, TestMacros, TestCoreAdminApis, ChaosMonkeySafeLeaderTest, LeaderElectionIntegrationTest, BufferStoreTest, TestNRTOpen, ShowFileRequestHandlerTest, DocValuesNotIndexedTest, TestRestoreCore, TestSha256AuthenticationProvider, TestMergePolicyConfig, TestDocSet, TestPKIAuthenticationPlugin, TestRandomFlRTGCloud, TestSearcherReuse, TestJsonFacets, SmileWriterTest, TestIndexSearcher, TestSolrCLIRunExample, TemplateUpdateProcessorTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, BasicZkTest, TestReplicationHandler, TestRandomDVFaceting, ZkCLITest, TestStressReorder, HardAutoCommitTest, SolrCoreTest, SpellCheckComponentTest, TestFunctionQuery, TestBadConfig, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, TestCoreDiscovery, SuggesterFSTTest, CoreAdminHandlerTest, TestTrie, TestCSVLoader, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, TestRemoteStreaming, TestSolrDeletionPolicy1, XsltUpdateRequestHandlerTest, IndexBasedSpellCheckerTest, TestWriterPerf, TestReversedWildcardFilterFactory, PrimitiveFieldTypeTest, TermsComponentTest, TestSolrQueryParser, LoggingHandlerTest, ReturnFieldsTest, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, SearchHandlerTest, HighlighterConfigTest, TestQuerySenderListener, UpdateParamsTest, AlternateDirectoryTest, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, TestSystemIdResolver, RAMDirectoryFactoryTest, TestSolrJ, TestUtils, TestDocumentBuilder, UUIDFieldTest, FileUtilsTest, CursorPagingTest, ActionThrottleTest, AssignTest, AsyncCallRequestStatusResponseTest, CdcrReplicationDistributedZkTest, CloudExitableDirectoryReaderTest, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, DistribJoinFromCollectionTest, DistributedQueueTest, DistributedVersionInfoTest, HttpPartitionTest, MissingSegmentRecoveryTest, OverseerModifyCollectionTest, OverseerTaskQueueTest, ReplicationFactorTest, RestartWhileUpdatingTest, RollingRestartTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestCloudInspectUtil, TestCloudPivotFacet]
   [junit4] Completed [507/691 (1!)] on J1 in 67.11s, 1 test, 1 error <<< FAILURES!

[...truncated 62909 lines...]
-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [echo] Checking for malformed docs...
     [exec] 
     [exec] /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/docs/solr-solrj/overview-summary.html
     [exec]   missing: org.apache.solr.client.solrj.io.eval
     [exec] 
     [exec] Missing javadocs were found!

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:775: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:654: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:670: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/common-build.xml:2490: exec returned: 1

Total time: 119 minutes 24 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 689 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/689/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

All tests passed

Build Log:
[...truncated 68254 lines...]
-ecj-javadoc-lint-tests:
    [mkdir] Created dir: /var/folders/qg/h2dfw5s161s51l2bn79mrb7r0000gn/T/ecj1767386041
 [ecj-lint] Compiling 759 source files to /var/folders/qg/h2dfw5s161s51l2bn79mrb7r0000gn/T/ecj1767386041
 [ecj-lint] invalid Class-Path header in manifest of jar file: /Users/jenkins/.ivy2/cache/org.restlet.jee/org.restlet/jars/org.restlet-2.3.0.jar
 [ecj-lint] invalid Class-Path header in manifest of jar file: /Users/jenkins/.ivy2/cache/org.restlet.jee/org.restlet.ext.servlet/jars/org.restlet.ext.servlet-2.3.0.jar
 [ecj-lint] ----------
 [ecj-lint] 1. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (at line 266)
 [ecj-lint] 	ZkStateReader reader = new ZkStateReader(zkClient);
 [ecj-lint] 	              ^^^^^^
 [ecj-lint] Resource leak: 'reader' is never closed
 [ecj-lint] ----------
 [ecj-lint] 2. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (at line 316)
 [ecj-lint] 	ZkStateReader reader = new ZkStateReader(zkClient);
 [ecj-lint] 	              ^^^^^^
 [ecj-lint] Resource leak: 'reader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 3. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java (at line 146)
 [ecj-lint] 	HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory();
 [ecj-lint] 	                     ^^^^^^^^^^^
 [ecj-lint] Resource leak: 'hdfsFactory' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 4. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java (at line 53)
 [ecj-lint] 	BasicAuthPlugin basicAuth = new BasicAuthPlugin();
 [ecj-lint] 	                ^^^^^^^^^
 [ecj-lint] Resource leak: 'basicAuth' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 5. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java (at line 163)
 [ecj-lint] 	SolrClient client = random().nextBoolean() ? collection1 : collection2;
 [ecj-lint] 	           ^^^^^^
 [ecj-lint] Resource leak: 'client' is never closed
 [ecj-lint] ----------
 [ecj-lint] 6. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java (at line 221)
 [ecj-lint] 	throw new AssertionError(q.toString() + ": " + e.getMessage(), e);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'client' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 7. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 201)
 [ecj-lint] 	Analyzer a1 = new WhitespaceAnalyzer();
 [ecj-lint] 	         ^^
 [ecj-lint] Resource leak: 'a1' is never closed
 [ecj-lint] ----------
 [ecj-lint] 8. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 204)
 [ecj-lint] 	OffsetWindowTokenFilter tots = new OffsetWindowTokenFilter(tokenStream);
 [ecj-lint] 	                        ^^^^
 [ecj-lint] Resource leak: 'tots' is never closed
 [ecj-lint] ----------
 [ecj-lint] 9. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/highlight/HighlighterTest.java (at line 208)
 [ecj-lint] 	Analyzer a2 = new WhitespaceAnalyzer();
 [ecj-lint] 	         ^^
 [ecj-lint] Resource leak: 'a2' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 10. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/request/TestBinaryResponseWriter.java (at line 65)
 [ecj-lint] 	NamedList res = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
 [ecj-lint] 	                            ^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 11. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/response/TestBinaryResponseWriter.java (at line 63)
 [ecj-lint] 	NamedList res = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
 [ecj-lint] 	                            ^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 12. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 72)
 [ecj-lint] 	JavabinTupleStreamParser parser = new JavabinTupleStreamParser(new ByteArrayInputStream(bytes), true);
 [ecj-lint] 	                         ^^^^^^
 [ecj-lint] Resource leak: 'parser' is never closed
 [ecj-lint] ----------
 [ecj-lint] 13. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 173)
 [ecj-lint] 	new JavaBinCodec().marshal(response.getValues(), baos);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] 14. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 175)
 [ecj-lint] 	Object o = new JavaBinCodec().unmarshal(new ByteArrayInputStream(bytes));
 [ecj-lint] 	           ^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] 15. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 178)
 [ecj-lint] 	JavabinTupleStreamParser parser = new JavabinTupleStreamParser(new ByteArrayInputStream(bytes), false);
 [ecj-lint] 	                         ^^^^^^
 [ecj-lint] Resource leak: 'parser' is never closed
 [ecj-lint] ----------
 [ecj-lint] 16. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/response/TestJavabinTupleStreamParser.java (at line 192)
 [ecj-lint] 	new JavaBinCodec().marshal(response.getValues(), baos);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 17. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/response/TestPushWriter.java (at line 57)
 [ecj-lint] 	m = (Map) new JavaBinCodec().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
 [ecj-lint] 	          ^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 18. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformer.java (at line 387)
 [ecj-lint] 	unmarshalled = (NamedList<Object>) new JavaBinCodec().unmarshal(
 [ecj-lint] 	                                   ^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 19. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/search/TestDocSet.java (at line 241)
 [ecj-lint] 	return loadfactor!=0 ? new HashDocSet(a,0,n,1/loadfactor) : new HashDocSet(a,0,n);
 [ecj-lint] 	                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] 20. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/search/TestDocSet.java (at line 531)
 [ecj-lint] 	DocSet a = new BitDocSet(bs);
 [ecj-lint] 	       ^
 [ecj-lint] Resource leak: 'a' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 21. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java (at line 85)
 [ecj-lint] 	final MockPKIAuthenticationPlugin mock = new MockPKIAuthenticationPlugin(null, nodeName);
 [ecj-lint] 	                                  ^^^^
 [ecj-lint] Resource leak: 'mock' is never closed
 [ecj-lint] ----------
 [ecj-lint] 22. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java (at line 138)
 [ecj-lint] 	MockPKIAuthenticationPlugin mock1 = new MockPKIAuthenticationPlugin(null, nodeName) {
 [ecj-lint] 	                            ^^^^^
 [ecj-lint] Resource leak: 'mock1' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 23. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/security/TestRuleBasedAuthorizationPlugin.java (at line 365)
 [ecj-lint] 	RuleBasedAuthorizationPlugin plugin = new RuleBasedAuthorizationPlugin();
 [ecj-lint] 	                             ^^^^^^
 [ecj-lint] Resource leak: 'plugin' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 24. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/security/TestSha256AuthenticationProvider.java (at line 49)
 [ecj-lint] 	BasicAuthPlugin basicAuthPlugin = new BasicAuthPlugin();
 [ecj-lint] 	                ^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'basicAuthPlugin' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 25. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java (at line 42)
 [ecj-lint] 	WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer();
 [ecj-lint] 	                   ^^^^^^^^
 [ecj-lint] Resource leak: 'analyzer' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 26. ERROR in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java (at line 22)
 [ecj-lint] 	import java.util.concurrent.atomic.AtomicInteger;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import java.util.concurrent.atomic.AtomicInteger is never used
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 27. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 139)
 [ecj-lint] 	IndexWriter w = new IndexWriter(d, newIndexWriterConfig(analyzer));
 [ecj-lint] 	            ^
 [ecj-lint] Resource leak: 'w' is never closed
 [ecj-lint] ----------
 [ecj-lint] 28. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 172)
 [ecj-lint] 	throw iae;
 [ecj-lint] 	^^^^^^^^^^
 [ecj-lint] Resource leak: 'w' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] 29. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java (at line 178)
 [ecj-lint] 	return;
 [ecj-lint] 	^^^^^^^
 [ecj-lint] Resource leak: 'w' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 30. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java (at line 415)
 [ecj-lint] 	new JavaBinCodec().marshal(topDocument, os);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] 31. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java (at line 419)
 [ecj-lint] 	SolrInputDocument result = (SolrInputDocument) new JavaBinCodec().unmarshal(is);
 [ecj-lint] 	                                               ^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 32. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (at line 181)
 [ecj-lint] 	EmbeddedSolrServer server1 = new EmbeddedSolrServer(h.getCoreContainer(), "split1");
 [ecj-lint] 	                   ^^^^^^^
 [ecj-lint] Resource leak: 'server1' is never closed
 [ecj-lint] ----------
 [ecj-lint] 33. WARNING in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (at line 182)
 [ecj-lint] 	EmbeddedSolrServer server2 = new EmbeddedSolrServer(h.getCoreContainer(), "split2");
 [ecj-lint] 	                   ^^^^^^^
 [ecj-lint] Resource leak: 'server2' is never closed
 [ecj-lint] ----------
 [ecj-lint] 33 problems (1 error, 32 warnings)

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:775: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:674: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/common-build.xml:2007: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/common-build.xml:2040: Compile failed; see the compiler error output for details.

Total time: 121 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 688 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/688/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.handler.admin.MBeansHandlerTest.testDiff

Error Message:
expected:<Was: [1, Now: 2], Delta: 1> but was:<Was: [4, Now: 5], Delta: 1>

Stack Trace:
org.junit.ComparisonFailure: expected:<Was: [1, Now: 2], Delta: 1> but was:<Was: [4, Now: 5], Delta: 1>
	at __randomizedtesting.SeedInfo.seed([A51E5D0D83429CD5:6008999693F4A4B5]:0)
	at org.junit.Assert.assertEquals(Assert.java:125)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.admin.MBeansHandlerTest.testDiff(MBeansHandlerTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12229 lines...]
   [junit4] Suite: org.apache.solr.handler.admin.MBeansHandlerTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.admin.MBeansHandlerTest_A51E5D0D83429CD5-001/init-core-data-001
   [junit4]   2> 2288686 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2288687 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 2288687 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2288727 WARN  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2288728 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2288752 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 2288808 WARN  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.s.IndexSchema [null] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2288812 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2288823 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 2288882 WARN  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2288884 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@58d59ef9
   [junit4]   2> 2288884 WARN  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2288887 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@58d59ef9
   [junit4]   2> 2288887 WARN  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2288887 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@58d59ef9
   [junit4]   2> 2288889 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib, /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2288938 WARN  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2288939 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.5.0
   [junit4]   2> 2288958 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2289021 WARN  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2289022 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2289031 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1
   [junit4]   2> 2289032 WARN  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2289032 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@58d59ef9
   [junit4]   2> 2289032 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/core/src/test-files/solr/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.admin.MBeansHandlerTest_A51E5D0D83429CD5-001/init-core-data-001/]
   [junit4]   2> 2289032 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@58d59ef9
   [junit4]   2> 2289033 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 2289132 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2289132 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2289133 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2289133 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2289133 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 2289134 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1bfa4514[collection1] main]
   [junit4]   2> 2289134 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 2289149 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 2289238 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2289238 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 2289239 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 2289239 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 2289239 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 2289239 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 2289239 INFO  (coreLoadExecutor-5952-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1558944697051774976
   [junit4]   2> 2289240 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 2289240 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 2289240 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 2289240 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 2289241 INFO  (searcherExecutor-5953-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1bfa4514[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2289245 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 2289247 INFO  (TEST-MBeansHandlerTest.testXMLDiffWithExternalEntity-seed#[A51E5D0D83429CD5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testXMLDiffWithExternalEntity
   [junit4]   2> 2289249 INFO  (TEST-MBeansHandlerTest.testXMLDiffWithExternalEntity-seed#[A51E5D0D83429CD5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testXMLDiffWithExternalEntity
   [junit4]   2> 2289251 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[A51E5D0D83429CD5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBackCompatNames
   [junit4]   2> 2289252 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[A51E5D0D83429CD5]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&wt=xml} status=0 QTime=1
   [junit4]   2> 2289260 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[A51E5D0D83429CD5]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&cat=QUERYHANDLER&wt=xml} status=0 QTime=0
   [junit4]   2> 2289261 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[A51E5D0D83429CD5]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&cat=UPDATE&wt=xml} status=0 QTime=0
   [junit4]   2> 2289261 INFO  (TEST-MBeansHandlerTest.testBackCompatNames-seed#[A51E5D0D83429CD5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBackCompatNames
   [junit4]   2> 2289267 INFO  (TEST-MBeansHandlerTest.testDiff-seed#[A51E5D0D83429CD5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDiff
   [junit4]   2> 2289270 INFO  (TEST-MBeansHandlerTest.testDiff-seed#[A51E5D0D83429CD5]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&stats=true&wt=xml} status=0 QTime=3
   [junit4]   2> 2289288 INFO  (TEST-MBeansHandlerTest.testDiff-seed#[A51E5D0D83429CD5]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={qt=/admin/mbeans&stats=true&diff=true&wt=xml} status=0 QTime=15
   [junit4]   2> 2289291 INFO  (TEST-MBeansHandlerTest.testDiff-seed#[A51E5D0D83429CD5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDiff
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=MBeansHandlerTest -Dtests.method=testDiff -Dtests.seed=A51E5D0D83429CD5 -Dtests.slow=true -Dtests.locale=et -Dtests.timezone=Europe/Moscow -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.03s J0 | MBeansHandlerTest.testDiff <<<
   [junit4]    > Throwable #1: org.junit.ComparisonFailure: expected:<Was: [1, Now: 2], Delta: 1> but was:<Was: [4, Now: 5], Delta: 1>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A51E5D0D83429CD5:6008999693F4A4B5]:0)
   [junit4]    > 	at org.apache.solr.handler.admin.MBeansHandlerTest.testDiff(MBeansHandlerTest.java:63)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2289296 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 2289296 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=456145362
   [junit4]   2> 2289297 INFO  (coreCloseExecutor-5958-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@2f39c1bb
   [junit4]   2> 2289323 INFO  (coreCloseExecutor-5958-thread-1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 2289326 INFO  (SUITE-MBeansHandlerTest-seed#[A51E5D0D83429CD5]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.handler.admin.MBeansHandlerTest_A51E5D0D83429CD5-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=735, maxMBSortInHeap=5.329176399897204, sim=RandomSimilarity(queryNorm=true,coord=yes): {}, locale=et, timezone=Europe/Moscow
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=316579008,total=426377216
   [junit4]   2> NOTE: All tests run in this JVM: [CdcrUpdateLogTest, TestAnalyzedSuggestions, DocExpirationUpdateProcessorFactoryTest, CdcrBootstrapTest, SegmentsInfoRequestHandlerTest, LeaderInitiatedRecoveryOnShardRestartTest, TestTolerantUpdateProcessorCloud, DeleteInactiveReplicaTest, TestReRankQParserPlugin, EnumFieldTest, LeaderElectionContextKeyTest, TestCloudDeleteByQuery, DocValuesMultiTest, TestRestManager, CollectionReloadTest, DateFieldTest, ResponseHeaderTest, TestDistribDocBasedVersion, JsonValidatorTest, DistribJoinFromCollectionTest, SolrGangliaReporterTest, TestExclusionRuleCollectionAccess, TestCryptoKeys, WrapperMergePolicyFactoryTest, HdfsDirectoryTest, TestApiFramework, AddBlockUpdateTest, DocValuesTest, HdfsChaosMonkeySafeLeaderTest, TestLegacyFieldCache, DeleteNodeTest, TestSort, TestDocTermOrds, TestFieldCacheReopen, DateRangeFieldTest, TestStressCloudBlindAtomicUpdates, TestImplicitCoreProperties, XmlUpdateRequestHandlerTest, TestAnalyzeInfixSuggestions, TestFieldCacheVsDocValues, TestFieldResource, TestConfigsApi, TestConfigSetsAPIExclusivity, DistributedQueryElevationComponentTest, ZkCLITest, ConnectionReuseTest, TestSchemaVersionResource, PluginInfoTest, TestJsonFacetRefinement, TestManagedResource, ShardRoutingTest, TestBinaryResponseWriter, HdfsDirectoryFactoryTest, TestPartialUpdateDeduplication, TestBM25SimilarityFactory, SpatialFilterTest, VMParamsZkACLAndCredentialsProvidersTest, TestManagedSchemaAPI, TestStressReorder, AliasIntegrationTest, PeerSyncWithIndexFingerprintCachingTest, TestFileDictionaryLookup, CoreAdminCreateDiscoverTest, TestElisionMultitermQuery, PolyFieldTest, TestDistributedStatsComponentCardinality, SolrCoreMetricManagerTest, ResponseLogComponentTest, TestTestInjection, LeaderFailureAfterFreshStartTest, FullHLLTest, PingRequestHandlerTest, SolrXmlInZkTest, RankQueryTest, TestUseDocValuesAsStored, TestBlobHandler, TestRandomRequestDistribution, TestPivotHelperCode, FileUtilsTest, UniqFieldsUpdateProcessorFactoryTest, ReplaceNodeTest, TestInitQParser, TestReqParamsAPI, DOMUtilTest, DistributedSpellCheckComponentTest, CreateCollectionCleanupTest, TestFastOutputStream, TestManagedSynonymFilterFactory, OverseerTest, OutputWriterTest, TestRequestStatusCollectionAPI, TestCoreContainer, TestLFUCache, SolrTestCaseJ4Test, TestUniqueKeyFieldResource, DirectUpdateHandlerTest, JSONWriterTest, PeerSyncTest, TestExactStatsCache, SharedFSAutoReplicaFailoverUtilsTest, RequiredFieldsTest, TestFastLRUCache, TestFieldTypeResource, TestPerFieldSimilarity, TestPushWriter, TermsComponentTest, ShardSplitTest, ParsingFieldUpdateProcessorsTest, TestPostingsSolrHighlighter, ZkStateWriterTest, SynonymTokenizerTest, HLLUtilTest, SoftAutoCommitTest, HdfsRecoveryZkTest, SOLR749Test, SimpleFacetsTest, MetricsHandlerTest, ShufflingReplicaListTransformerTest, TestRuleBasedAuthorizationPlugin, CoreMergeIndexesAdminHandlerTest, TestSchemaResource, TestSolrFieldCacheMBean, BlobRepositoryCloudTest, TestLeaderElectionWithEmptyReplica, SolrMetricReporterTest, TestIBSimilarityFactory, TestRandomDVFaceting, OverseerRolesTest, DirectSolrSpellCheckerTest, BlockJoinFacetSimpleTest, TestXmlQParser, SystemInfoHandlerTest, HdfsWriteToMultipleCollectionsTest, BigEndianAscendingWordDeserializerTest, TestSolrJ, TestNamedUpdateProcessors, TestInitParams, BitVectorTest, TestRTGBase, TestUnifiedSolrHighlighter, BasicAuthIntegrationTest, TestSolrIndexConfig, TestSolrCoreSnapshots, DocumentBuilderTest, TestSolrQueryParserDefaultOperatorResource, InfixSuggestersTest, DirectoryFactoryTest, TestHdfsUpdateLog, TestHdfsBackupRestoreCore, SolrCLIZkUtilsTest, DistributedDebugComponentTest, TestSolrXml, TestSizeLimitedDistributedMap, TestRandomCollapseQParserPlugin, SmileWriterTest, TestComponentsName, LeaderElectionTest, UnloadDistributedZkTest, TestNumericTerms64, AnalyticsMergeStrategyTest, SolrJmxReporterTest, TestReplicaProperties, TestSubQueryTransformerCrossCore, TestHighFrequencyDictionaryFactory, DistributedFacetPivotSmallTest, TestDistributedSearch, ConfigSetsAPITest, TestSweetSpotSimilarityFactory, UUIDUpdateProcessorFallbackTest, TestRealTimeGet, TestJoin, TestRangeQuery, TestSolr4Spatial, TestGroupingSearch, QueryElevationComponentTest, HighlighterTest, ShowFileRequestHandlerTest, CurrencyFieldOpenExchangeTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, SimplePostToolTest, AnalysisAfterCoreReloadTest, TestExtendedDismaxParser, SuggesterFSTTest, SuggesterTSTTest, TestStressLucene, TestTrie, SchemaVersionSpecificBehaviorTest, TestWordDelimiterFilterFactory, StatelessScriptUpdateProcessorFactoryTest, XsltUpdateRequestHandlerTest, CacheHeaderTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, StandardRequestHandlerTest, TestWriterPerf, TestQueryTypes, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, MoreLikeThisHandlerTest, FastVectorHighlighterTest, SolrPluginUtilsTest, TestCollationField, ReturnFieldsTest, MBeansHandlerTest]
   [junit4] Completed [424/691 (1!)] on J0 in 0.66s, 3 tests, 1 failure <<< FAILURES!

[...truncated 63167 lines...]
-documentation-lint:
    [jtidy] Checking for broken html (such as invalid tags)...
   [delete] Deleting directory /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/build/jtidy_tmp
     [echo] Checking for broken links...
     [exec] 
     [exec] Crawl/parse...
     [exec] 
     [exec] Verify...
     [echo] Checking for malformed docs...
     [exec] 
     [exec] /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/docs/solr-solrj/overview-summary.html
     [exec]   missing: org.apache.solr.client.solrj.io.eval
     [exec] 
     [exec] Missing javadocs were found!

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:775: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:654: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build.xml:670: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/lucene/common-build.xml:2490: exec returned: 1

Total time: 118 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any