You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/09/19 19:17:55 UTC

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk1.8.0_144) - Build # 20500 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/20500/
Java: 64bit/jdk1.8.0_144 -XX:-UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:     1) Thread[id=16561, name=searcherExecutor-4179-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   1) Thread[id=16561, name=searcherExecutor-4179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([D03E314842FD5B6E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=16561, name=searcherExecutor-4179-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=16561, name=searcherExecutor-4179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([D03E314842FD5B6E]:0)


FAILED:  org.apache.solr.core.TestLazyCores.testNoCommit

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([D03E314842FD5B6E:F5E909989DA38CB]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:884)
	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:847)
	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:829)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">2</int><lst name="params"><str name="q">*:*</str></lst></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=*:*
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:877)
	... 41 more




Build Log:
[...truncated 11934 lines...]
   [junit4] Suite: org.apache.solr.core.TestLazyCores
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/init-core-data-001
   [junit4]   2> 766565 WARN  (SUITE-TestLazyCores-seed#[D03E314842FD5B6E]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1538 numCloses=1538
   [junit4]   2> 766566 INFO  (SUITE-TestLazyCores-seed#[D03E314842FD5B6E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 766567 INFO  (SUITE-TestLazyCores-seed#[D03E314842FD5B6E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 766567 INFO  (SUITE-TestLazyCores-seed#[D03E314842FD5B6E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 766568 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCachingLimit
   [junit4]   2> 766576 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/solr.xml
   [junit4]   2> 766578 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 766607 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766611 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766611 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766613 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 766619 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 766619 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 766619 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 766631 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 766631 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection2, trusted=true
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection1, trusted=true
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection2' (registry 'solr.core.collection2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection2], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection2/data/]
   [junit4]   2> 766633 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection1], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection1/data/]
   [junit4]   2> 766635 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 766635 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1460892262, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 766635 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection5, trusted=true
   [junit4]   2> 766635 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1460892262, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 766635 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection5' (registry 'solr.core.collection5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766635 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 766635 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection5], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection5/data/]
   [junit4]   2> 766637 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1460892262, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 766673 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 766673 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 766673 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=216370651, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 766673 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 766673 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 766674 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@54bc94a6[collection5] main]
   [junit4]   2> 766674 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection5/conf
   [junit4]   2> 766674 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=216370651, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 766674 INFO  (coreLoadExecutor-4021-thread-3) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 766674 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@5a7ffa1a[collection2] main]
   [junit4]   2> 766674 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection2/conf
   [junit4]   2> 766675 INFO  (coreLoadExecutor-4021-thread-2) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 766675 INFO  (searcherExecutor-4024-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@54bc94a6[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 766675 INFO  (searcherExecutor-4022-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@5a7ffa1a[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 766678 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 766679 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 766679 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=216370651, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 766680 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6daf1667[collection1] main]
   [junit4]   2> 766680 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection1/conf
   [junit4]   2> 766680 INFO  (coreLoadExecutor-4021-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 766681 INFO  (searcherExecutor-4023-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6daf1667[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 766686 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 766690 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
   [junit4]   2> 766703 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 766703 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection3, trusted=true
   [junit4]   2> 766704 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection3' (registry 'solr.core.collection3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766704 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 766704 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection3], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection3/data/]
   [junit4]   2> 766705 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1186827766, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 766746 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 766746 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 766747 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=69.0693359375, floorSegmentMB=0.8564453125, forceMergeDeletesPctAllowed=18.775439600677043, segmentsPerTier=15.0, maxCFSSegmentSizeMB=0.828125, noCFSRatio=0.12411374803329754
   [junit4]   2> 766747 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@2be7cda9[collection3] main]
   [junit4]   2> 766747 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection3/conf
   [junit4]   2> 766748 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 766748 INFO  (searcherExecutor-4034-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@2be7cda9[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 766753 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 766757 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema [collection4] Schema name=tiny
   [junit4]   2> 766758 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 766758 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection4, trusted=true
   [junit4]   2> 766758 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection4' (registry 'solr.core.collection4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766759 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 766759 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection4], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection4/data/]
   [junit4]   2> 766760 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1996962121, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 766793 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 766793 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 766793 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=88.302734375, floorSegmentMB=0.4375, forceMergeDeletesPctAllowed=4.954997154046757, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 766794 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@378d87c8[collection4] main]
   [junit4]   2> 766794 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection4/conf
   [junit4]   2> 766794 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 766795 INFO  (searcherExecutor-4038-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@378d87c8[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 766800 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 766816 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema [collection6] Schema name=tiny
   [junit4]   2> 766818 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 766818 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection6' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection6, trusted=true
   [junit4]   2> 766818 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection6' (registry 'solr.core.collection6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766818 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 766818 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [[collection6] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection6], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection6/data/]
   [junit4]   2> 766820 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=5.4775390625, floorSegmentMB=2.041015625, forceMergeDeletesPctAllowed=22.48634943510553, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.43401061629073956
   [junit4]   2> 766858 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 766858 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 766858 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=46.1689453125, floorSegmentMB=1.701171875, forceMergeDeletesPctAllowed=13.308777170388744, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.25870716514526154
   [junit4]   2> 766859 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@713ce8ef[collection6] main]
   [junit4]   2> 766859 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection6/conf
   [junit4]   2> 766859 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 766860 INFO  (searcherExecutor-4042-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection6] Registered new searcher Searcher@713ce8ef[collection6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 766865 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 766868 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema [collection7] Schema name=tiny
   [junit4]   2> 766869 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 766869 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection7' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection7, trusted=true
   [junit4]   2> 766870 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection7' (registry 'solr.core.collection7') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766870 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 766870 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [[collection7] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection7], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection7/data/]
   [junit4]   2> 766872 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=11.330078125, floorSegmentMB=1.46484375, forceMergeDeletesPctAllowed=15.535024959046433, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.572665239957295
   [junit4]   2> 766926 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 766926 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 766928 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=96.1953125, floorSegmentMB=1.572265625, forceMergeDeletesPctAllowed=29.988447818475127, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6213514017369839
   [junit4]   2> 766929 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@1f096224[collection7] main]
   [junit4]   2> 766929 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection7/conf
   [junit4]   2> 766930 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 766930 INFO  (searcherExecutor-4046-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection7] Registered new searcher Searcher@1f096224[collection7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 766939 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 766961 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema [collection8] Schema name=tiny
   [junit4]   2> 766963 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 766963 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection8' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection8, trusted=true
   [junit4]   2> 766964 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection8' (registry 'solr.core.collection8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 766964 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 766964 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [[collection8] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection8], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection8/data/]
   [junit4]   2> 766966 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 767011 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 767011 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 767012 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=28, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.116050772203418]
   [junit4]   2> 767013 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@139d59f[collection8] main]
   [junit4]   2> 767013 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection8/conf
   [junit4]   2> 767014 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 767014 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.TransientSolrCoreCacheDefault Closing transient core [collection3]
   [junit4]   2> 767015 INFO  (searcherExecutor-4050-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection8] Registered new searcher Searcher@139d59f[collection8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 767022 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 767026 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema [collection9] Schema name=tiny
   [junit4]   2> 767027 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 767027 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection9' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection9, trusted=true
   [junit4]   2> 767028 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection9' (registry 'solr.core.collection9') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767028 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 767028 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [[collection9] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection9], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/./collection9/data/]
   [junit4]   2> 767029 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=59.7138671875, floorSegmentMB=0.74609375, forceMergeDeletesPctAllowed=6.7829184734026455, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 767072 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 767072 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 767073 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=7.0087890625, floorSegmentMB=0.6728515625, forceMergeDeletesPctAllowed=1.4824611569912172, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 767073 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@2bb5c9a8[collection9] main]
   [junit4]   2> 767074 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-001/collection9/conf
   [junit4]   2> 767074 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 767074 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.TransientSolrCoreCacheDefault Closing transient core [collection2]
   [junit4]   2> 767074 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@2ef487c3
   [junit4]   2> 767074 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=787777475
   [junit4]   2> 767075 INFO  (searcherExecutor-4054-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection9] Registered new searcher Searcher@2bb5c9a8[collection9] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 767075 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6fb2564c: rootName = null, domain = solr.core.collection2, service url = null, agent id = null] for registry solr.core.collection2 / com.codahale.metrics.MetricRegistry@3bfe74e6
   [junit4]   2> 767085 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [collection3]  CLOSING SolrCore org.apache.solr.core.SolrCore@2be07b7c
   [junit4]   2> 767085 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection3, tag=736131964
   [junit4]   2> 767085 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6672cd90: rootName = null, domain = solr.core.collection3, service url = null, agent id = null] for registry solr.core.collection3 / com.codahale.metrics.MetricRegistry@62792452
   [junit4]   2> 767094 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Shutting down CoreContainer instance=597547980
   [junit4]   2> 767095 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 767095 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@ca7514c: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@30e9f3b0
   [junit4]   2> 767099 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 767099 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@b66c2e4: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5b58a8ac
   [junit4]   2> 767103 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 767103 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5bed664b: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@2f286e5f
   [junit4]   2> 767103 INFO  (coreCloseExecutor-4058-thread-1-processing-x:collection3) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@4c5402a9
   [junit4]   2> 767103 INFO  (coreCloseExecutor-4058-thread-1-processing-x:collection3) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=1280574121
   [junit4]   2> 767103 INFO  (coreCloseExecutor-4058-thread-2-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3c702dd2
   [junit4]   2> 767103 INFO  (coreCloseExecutor-4058-thread-1-processing-x:collection3) [    x:collection5] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3c5de9f8: rootName = null, domain = solr.core.collection5, service url = null, agent id = null] for registry solr.core.collection5 / com.codahale.metrics.MetricRegistry@701f19b8
   [junit4]   2> 767105 INFO  (coreCloseExecutor-4058-thread-3-processing-x:collection3) [    x:collection4] o.a.s.c.SolrCore [collection4]  CLOSING SolrCore org.apache.solr.core.SolrCore@53158480
   [junit4]   2> 767106 INFO  (coreCloseExecutor-4058-thread-4-processing-x:collection3) [    x:collection6] o.a.s.c.SolrCore [collection6]  CLOSING SolrCore org.apache.solr.core.SolrCore@2083bd4c
   [junit4]   2> 767106 INFO  (coreCloseExecutor-4058-thread-5-processing-x:collection3) [    x:collection7] o.a.s.c.SolrCore [collection7]  CLOSING SolrCore org.apache.solr.core.SolrCore@1dd1a983
   [junit4]   2> 767106 INFO  (coreCloseExecutor-4058-thread-6-processing-x:collection3) [    x:collection8] o.a.s.c.SolrCore [collection8]  CLOSING SolrCore org.apache.solr.core.SolrCore@30a66083
   [junit4]   2> 767108 INFO  (coreCloseExecutor-4058-thread-2-processing-x:collection3) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1013984722
   [junit4]   2> 767108 INFO  (coreCloseExecutor-4058-thread-7-processing-x:collection3) [    x:collection9] o.a.s.c.SolrCore [collection9]  CLOSING SolrCore org.apache.solr.core.SolrCore@1b641a5e
   [junit4]   2> 767108 INFO  (coreCloseExecutor-4058-thread-2-processing-x:collection3) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@715ee83b: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@6733cbb
   [junit4]   2> 767124 INFO  (coreCloseExecutor-4058-thread-3-processing-x:collection3) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection4, tag=1393919104
   [junit4]   2> 767124 INFO  (coreCloseExecutor-4058-thread-3-processing-x:collection3) [    x:collection4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6192254e: rootName = null, domain = solr.core.collection4, service url = null, agent id = null] for registry solr.core.collection4 / com.codahale.metrics.MetricRegistry@59cf0d75
   [junit4]   2> 767129 INFO  (coreCloseExecutor-4058-thread-4-processing-x:collection3) [    x:collection6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection6, tag=545504588
   [junit4]   2> 767129 INFO  (coreCloseExecutor-4058-thread-4-processing-x:collection3) [    x:collection6] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3f70ddaa: rootName = null, domain = solr.core.collection6, service url = null, agent id = null] for registry solr.core.collection6 / com.codahale.metrics.MetricRegistry@d760e2e
   [junit4]   2> 767136 INFO  (coreCloseExecutor-4058-thread-5-processing-x:collection3) [    x:collection7] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection7, tag=500279683
   [junit4]   2> 767136 INFO  (coreCloseExecutor-4058-thread-5-processing-x:collection3) [    x:collection7] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5971b87c: rootName = null, domain = solr.core.collection7, service url = null, agent id = null] for registry solr.core.collection7 / com.codahale.metrics.MetricRegistry@76072e00
   [junit4]   2> 767142 INFO  (coreCloseExecutor-4058-thread-6-processing-x:collection3) [    x:collection8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection8, tag=816210051
   [junit4]   2> 767142 INFO  (coreCloseExecutor-4058-thread-6-processing-x:collection3) [    x:collection8] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@79abc75b: rootName = null, domain = solr.core.collection8, service url = null, agent id = null] for registry solr.core.collection8 / com.codahale.metrics.MetricRegistry@37d816e0
   [junit4]   2> 767149 INFO  (coreCloseExecutor-4058-thread-7-processing-x:collection3) [    x:collection9] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection9, tag=459545182
   [junit4]   2> 767149 INFO  (coreCloseExecutor-4058-thread-7-processing-x:collection3) [    x:collection9] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2e318bfd: rootName = null, domain = solr.core.collection9, service url = null, agent id = null] for registry solr.core.collection9 / com.codahale.metrics.MetricRegistry@7dd5a063
   [junit4]   2> 767155 INFO  (TEST-TestLazyCores.testCachingLimit-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Ending testCachingLimit
   [junit4]   2> 767157 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Starting testLazyLoad
   [junit4]   2> 767168 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/solr.xml
   [junit4]   2> 767172 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 767201 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767209 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767209 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767216 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 767224 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 767225 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 767248 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 767248 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 767249 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 767249 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection1, trusted=true
   [junit4]   2> 767250 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767250 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 767250 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection1], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection1/data/]
   [junit4]   2> 767250 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 767250 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection2, trusted=true
   [junit4]   2> 767251 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection2' (registry 'solr.core.collection2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767251 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 767251 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection2], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection2/data/]
   [junit4]   2> 767252 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1460892262, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 767253 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1460892262, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 767257 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 767262 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 767264 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 767264 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection5, trusted=true
   [junit4]   2> 767264 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection5' (registry 'solr.core.collection5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767264 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 767264 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection5], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection5/data/]
   [junit4]   2> 767268 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1460892262, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 767309 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 767310 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 767311 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=216370651, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 767311 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@6fdb50a[collection2] main]
   [junit4]   2> 767311 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection2/conf
   [junit4]   2> 767312 INFO  (coreLoadExecutor-4065-thread-2-processing-x:collection3) [    x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 767313 INFO  (searcherExecutor-4067-thread-1-processing-x:collection2) [    x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@6fdb50a[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 767318 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 767318 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 767319 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=216370651, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 767319 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@17d4fe90[collection1] main]
   [junit4]   2> 767319 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection1/conf
   [junit4]   2> 767320 INFO  (coreLoadExecutor-4065-thread-1-processing-x:collection3) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 767320 INFO  (searcherExecutor-4066-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@17d4fe90[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 767329 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 767329 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 767330 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=216370651, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 767330 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@136a4a4c[collection5] main]
   [junit4]   2> 767331 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection5/conf
   [junit4]   2> 767331 INFO  (coreLoadExecutor-4065-thread-3-processing-x:collection3) [    x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 767332 INFO  (searcherExecutor-4068-thread-1-processing-x:collection5) [    x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@136a4a4c[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 767362 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 767366 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
   [junit4]   2> 767368 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 767368 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection3, trusted=true
   [junit4]   2> 767369 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection3' (registry 'solr.core.collection3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767369 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 767369 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection3], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection3/data/]
   [junit4]   2> 767372 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=49.6474609375, floorSegmentMB=1.1787109375, forceMergeDeletesPctAllowed=18.283424549859078, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 767438 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 767438 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 767438 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=4.181640625, floorSegmentMB=0.6279296875, forceMergeDeletesPctAllowed=20.02444742389012, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.12637187746336329
   [junit4]   2> 767439 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@7b33429c[collection3] main]
   [junit4]   2> 767439 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection3/conf
   [junit4]   2> 767440 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 767440 INFO  (searcherExecutor-4078-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@7b33429c[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 767446 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 767453 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema [collection4] Schema name=tiny
   [junit4]   2> 767457 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 767457 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection4, trusted=true
   [junit4]   2> 767458 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection4' (registry 'solr.core.collection4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767458 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 767458 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection4], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/./collection4/data/]
   [junit4]   2> 767460 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=32.0615234375, floorSegmentMB=0.267578125, forceMergeDeletesPctAllowed=26.08483334925852, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4638618587415254
   [junit4]   2> 767545 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 767546 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 767549 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=88.4296875, floorSegmentMB=1.509765625, forceMergeDeletesPctAllowed=26.88047747219356, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 767549 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@6b8f4ebb[collection4] main]
   [junit4]   2> 767550 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-002/collection4/conf
   [junit4]   2> 767550 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 767551 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.CoreContainer Shutting down CoreContainer instance=706708875
   [junit4]   2> 767551 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 767551 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7dccec7f: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@3ef69dd3
   [junit4]   2> 767551 INFO  (searcherExecutor-4082-thread-1-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@6b8f4ebb[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 767556 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 767556 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@46f02f36: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5b58a8ac
   [junit4]   2> 767559 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 767560 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@39b03da5: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@2f286e5f
   [junit4]   2> 767560 INFO  (coreCloseExecutor-4086-thread-2-processing-x:collection3) [    x:collection5] o.a.s.c.SolrCore [collection5]  CLOSING SolrCore org.apache.solr.core.SolrCore@1ca1f59d
   [junit4]   2> 767560 INFO  (coreCloseExecutor-4086-thread-3-processing-x:collection3) [    x:collection4] o.a.s.c.SolrCore [collection4]  CLOSING SolrCore org.apache.solr.core.SolrCore@7ed4b703
   [junit4]   2> 767560 INFO  (coreCloseExecutor-4086-thread-2-processing-x:collection3) [    x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=480376221
   [junit4]   2> 767560 INFO  (coreCloseExecutor-4086-thread-2-processing-x:collection3) [    x:collection5] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7c379fe4: rootName = null, domain = solr.core.collection5, service url = null, agent id = null] for registry solr.core.collection5 / com.codahale.metrics.MetricRegistry@584f6e78
   [junit4]   2> 767567 INFO  (coreCloseExecutor-4086-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@54843e32
   [junit4]   2> 767567 INFO  (coreCloseExecutor-4086-thread-1-processing-x:collection3) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1417952818
   [junit4]   2> 767567 INFO  (coreCloseExecutor-4086-thread-1-processing-x:collection3) [    x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2b29849c: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@2d1ec3c6
   [junit4]   2> 767573 INFO  (coreCloseExecutor-4086-thread-4-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCore [collection2]  CLOSING SolrCore org.apache.solr.core.SolrCore@220ea14a
   [junit4]   2> 767574 INFO  (coreCloseExecutor-4086-thread-4-processing-x:collection3) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=571384138
   [junit4]   2> 767574 INFO  (coreCloseExecutor-4086-thread-4-processing-x:collection3) [    x:collection2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@166a1029: rootName = null, domain = solr.core.collection2, service url = null, agent id = null] for registry solr.core.collection2 / com.codahale.metrics.MetricRegistry@3f04f0eb
   [junit4]   2> 767574 INFO  (coreCloseExecutor-4086-thread-5-processing-x:collection3) [    x:collection3] o.a.s.c.SolrCore [collection3]  CLOSING SolrCore org.apache.solr.core.SolrCore@15ac9819
   [junit4]   2> 767582 INFO  (coreCloseExecutor-4086-thread-3-processing-x:collection3) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection4, tag=2127869699
   [junit4]   2> 767582 INFO  (coreCloseExecutor-4086-thread-3-processing-x:collection3) [    x:collection4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@794d3c3c: rootName = null, domain = solr.core.collection4, service url = null, agent id = null] for registry solr.core.collection4 / com.codahale.metrics.MetricRegistry@4a3df40b
   [junit4]   2> 767593 INFO  (coreCloseExecutor-4086-thread-5-processing-x:collection3) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection3, tag=363632665
   [junit4]   2> 767593 INFO  (coreCloseExecutor-4086-thread-5-processing-x:collection3) [    x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@539b5840: rootName = null, domain = solr.core.collection3, service url = null, agent id = null] for registry solr.core.collection3 / com.codahale.metrics.MetricRegistry@36db8335
   [junit4]   2> 767615 INFO  (TEST-TestLazyCores.testLazyLoad-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Ending testLazyLoad
   [junit4]   2> 767617 INFO  (TEST-TestLazyCores.testRace-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Starting testRace
   [junit4]   2> 767627 INFO  (TEST-TestLazyCores.testRace-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-003/solr.xml
   [junit4]   2> 767631 INFO  (TEST-TestLazyCores.testRace-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 767662 INFO  (TEST-TestLazyCores.testRace-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767671 INFO  (TEST-TestLazyCores.testRace-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767671 INFO  (TEST-TestLazyCores.testRace-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767681 INFO  (TEST-TestLazyCores.testRace-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 767690 INFO  (coreLoadExecutor-4093-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 767693 INFO  (coreLoadExecutor-4093-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 767724 INFO  (coreLoadExecutor-4093-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
   [junit4]   2> 767727 INFO  (coreLoadExecutor-4093-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 767727 INFO  (coreLoadExecutor-4093-thread-2-processing-x:collection3) [    x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 767727 INFO  (coreLoadExecutor-4093-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-003/./collection2, trusted=true
   [junit4]   2> 767728 INFO  (coreLoadExecutor-4093-thread-2-processing-x:collection3) [    x:collection2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection2' (registry 'solr.core.collection2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767728 INFO  (coreLoadExecutor-4093-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 767729 INFO  (coreLoadExecutor-4093-thread-2-processing-x:collection3) [    x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-003/collection2], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-003/./collection2/data/]
   [junit4]   2> 767732 INFO  (coreLoadExecutor-4093-thread-2-processing-x:collection3) [    x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1460892262, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 767732 INFO  (coreLoadExecutor-4093-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
   [junit4]   2> 767734 INFO  (coreLoadExecutor-4093-thread-1-processing-x:collection3) [    x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 767734 INFO  (coreLoadExecutor-4093-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-003/./collection1, trusted=true
   [junit4]   2> 767734 INFO  (coreLoadExecutor-4093-thread-1-processing-x:collection3) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@172a8ac
   [junit4]   2> 767734 INFO  (coreLoadExecutor-4093-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 767734 INFO  (coreLoadExecutor-4093-thread-1-processing-x:collection3) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-003/collection1], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001/tempDir-003/./collection1/data/]
   [junit4]   2> 767736 INFO  (coreLoadExecutor-4093-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
   [junit4]   2> 767737 INFO  (coreLoadExecutor-4093-thread-1-processing-x:collection3) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1460892262, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 767738 INFO  (coreLoadExecutor-4093-thread-3-processing-x:collection3) [    x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
   [junit4]   2> 767738 INFO  (coreLoadExecutor-4093-thread-3-processing-x:collection3) [    x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /home/jenkins/worksp

[...truncated too long message...]

s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5beccdd8: rootName = null, domain = solr.core.collection5, service url = null, agent id = null] for registry solr.core.collection5 / com.codahale.metrics.MetricRegistry@6a3baafc
   [junit4]   2> 777029 INFO  (coreCloseExecutor-4342-thread-2-processing-x:collection3) [    x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=1138421633
   [junit4]   2> 777029 INFO  (coreCloseExecutor-4342-thread-2-processing-x:collection3) [    x:collection2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@39692d84: rootName = null, domain = solr.core.collection2, service url = null, agent id = null] for registry solr.core.collection2 / com.codahale.metrics.MetricRegistry@36355c7b
   [junit4]   2> 777039 INFO  (TEST-TestLazyCores.testMidUseUnload-seed#[D03E314842FD5B6E]) [    x:collection3] o.a.s.SolrTestCaseJ4 ###Ending testMidUseUnload
   [junit4]   2> 777040 INFO  (SUITE-TestLazyCores-seed#[D03E314842FD5B6E]-worker) [    x:collection3] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 777040 INFO  (SUITE-TestLazyCores-seed#[D03E314842FD5B6E]-worker) [    x:collection3] o.a.s.c.CoreContainer Shutting down CoreContainer instance=782485950
   [junit4]   2> 777040 INFO  (SUITE-TestLazyCores-seed#[D03E314842FD5B6E]-worker) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 777040 INFO  (SUITE-TestLazyCores-seed#[D03E314842FD5B6E]-worker) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 777040 INFO  (SUITE-TestLazyCores-seed#[D03E314842FD5B6E]-worker) [    x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J0/temp/solr.core.TestLazyCores_D03E314842FD5B6E-001
   [junit4]   2> Sep 19, 2017 7:00:02 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Sep 19, 2017 7:00:22 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]   2>    1) Thread[id=16561, name=searcherExecutor-4179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Sep 19, 2017 7:00:22 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=16561, name=searcherExecutor-4179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2> Sep 19, 2017 7:00:25 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=16561, name=searcherExecutor-4179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {v_t=BlockTreeOrds(blocksize=128), id=BlockTreeOrds(blocksize=128), v_s=PostingsFormat(name=Memory), v_f=PostingsFormat(name=MockRandom)}, docValues:{}, maxPointsInLeafNode=215, maxMBSortInHeap=5.631583052798675, sim=RandomSimilarity(queryNorm=false): {}, locale=en-ZA, timezone=SystemV/HST10
   [junit4]   2> NOTE: Linux 4.10.0-33-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=8,threads=2,free=75389072,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestImplicitCoreProperties, TestCorePropertiesReload, HdfsChaosMonkeyNothingIsSafeTest, ChaosMonkeyNothingIsSafeTest, TestScoreJoinQPScore, TestWriterPerf, TestLegacyTerms, AddSchemaFieldsUpdateProcessorFactoryTest, TestConfig, TestTolerantUpdateProcessorRandomCloud, TestNumericTerms64, TestExceedMaxTermLength, TestDeleteCollectionOnDownNodes, PeerSyncTest, RecoveryAfterSoftCommitTest, HdfsNNFailoverTest, TestHighFrequencyDictionaryFactory, TestLeaderElectionZkExpiry, OverseerStatusTest, TestLuceneMatchVersion, ConfigSetsAPITest, WordBreakSolrSpellCheckerTest, TestBulkSchemaAPI, TestInitQParser, AsyncCallRequestStatusResponseTest, TestRangeQuery, TestSimpleTextCodec, TestJoin, TestIndexingPerformance, SearchHandlerTest, BlockJoinFacetRandomTest, SpatialRPTFieldTypeTest, TestBadConfig, AliasIntegrationTest, TestCloudJSONFacetJoinDomain, TestCloudRecovery, TestRandomFaceting, AnalysisErrorHandlingTest, TestMergePolicyConfig, BinaryUpdateRequestHandlerTest, TestManagedResource, TestConfigSetImmutable, CustomCollectionTest, TestRandomDVFaceting, TestPhraseSuggestions, TestDefaultStatsCache, TestFieldCacheWithThreads, TestMacros, TestPivotHelperCode, TestManagedSynonymFilterFactory, TestIntervalFaceting, MBeansHandlerTest, MigrateRouteKeyTest, TestRemoteStreaming, TestRandomFlRTGCloud, TestPolicyCloud, DefaultValueUpdateProcessorTest, RAMDirectoryFactoryTest, TestComponentsName, TestHdfsBackupRestoreCore, HLLUtilTest, SpellCheckCollatorTest, ReplicationFactorTest, TestSolrConfigHandlerCloud, TestStressLiveNodes, DocValuesNotIndexedTest, TestLockTree, MetricsHandlerTest, SpatialHeatmapFacetsTest, SolrRequestParserTest, DirectoryFactoryTest, TestMissingGroups, ZkStateWriterTest, TestSearcherReuse, TestStressLucene, SolrGangliaReporterTest, ExplicitHLLTest, TestReRankQParserPlugin, TestAuthenticationFramework, CursorPagingTest, TestExactStatsCache, TestRealTimeGet, BufferStoreTest, TestFaceting, TestSolrDeletionPolicy1, TestInPlaceUpdatesStandalone, TestLazyCores]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLazyCores -Dtests.seed=D03E314842FD5B6E -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=en-ZA -Dtests.timezone=SystemV/HST10 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestLazyCores (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]    >    1) Thread[id=16561, name=searcherExecutor-4179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D03E314842FD5B6E]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=16561, name=searcherExecutor-4179-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D03E314842FD5B6E]:0)
   [junit4] Completed [243/733 (1!)] on J0 in 33.59s, 9 tests, 3 errors <<< FAILURES!

[...truncated 47717 lines...]

[JENKINS-EA] Lucene-Solr-master-Linux (64bit/jdk-9-ea+181) - Build # 20502 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/20502/
Java: 64bit/jdk-9-ea+181 -XX:-UseCompressedOops -XX:+UseG1GC --illegal-access=deny

1 tests failed.
FAILED:  org.apache.solr.update.TestInPlaceUpdatesDistrib.test

Error Message:
inplace_updatable_float didn't match for replica at client: https://127.0.0.1:38277/collection1 expected:<109.0> but was:<1.0>

Stack Trace:
java.lang.AssertionError: inplace_updatable_float didn't match for replica at client: https://127.0.0.1:38277/collection1 expected:<109.0> but was:<1.0>
	at __randomizedtesting.SeedInfo.seed([9C854BAF3D78B37E:14D174759384DE86]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.apache.solr.update.TestInPlaceUpdatesDistrib.assertReplicaValue(TestInPlaceUpdatesDistrib.java:980)
	at org.apache.solr.update.TestInPlaceUpdatesDistrib.outOfOrderUpdatesIndividualReplicaTest(TestInPlaceUpdatesDistrib.java:643)
	at org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:144)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)




Build Log:
[...truncated 11265 lines...]
   [junit4] Suite: org.apache.solr.update.TestInPlaceUpdatesDistrib
   [junit4]   2> 28143 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/init-core-data-001
   [junit4]   2> 28144 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 28144 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 28150 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 28151 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 28151 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 28167 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 28244 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=inplace-updates
   [junit4]   2> 28247 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
   [junit4]   2> 28313 WARN  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
   [junit4]   2> 28313 WARN  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
   [junit4]   2> 28313 WARN  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
   [junit4]   2> 28334 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 28367 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 28375 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.s.IndexSchema [collection1] Schema name=inplace-updates
   [junit4]   2> 28378 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
   [junit4]   2> 28378 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1, trusted=true
   [junit4]   2> 28378 WARN  (coreLoadExecutor-261-thread-1) [    ] o.a.s.m.r.SolrJmxReporter No JMX server found. Not exposing Solr metrics via JMX.
   [junit4]   2> 28378 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 28379 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/init-core-data-001/]
   [junit4]   2> 28382 WARN  (coreLoadExecutor-261-thread-1) [    ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 28462 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 28462 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 28463 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 28463 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 28464 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@49548b1f[collection1] main]
   [junit4]   2> 28464 WARN  (coreLoadExecutor-261-thread-1) [    ] o.a.s.r.ManagedResourceStorage Cannot write to config directory /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 28464 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 28465 INFO  (coreLoadExecutor-261-thread-1) [    ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579029133223002112
   [junit4]   2> 28467 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 28475 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 28479 INFO  (searcherExecutor-262-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@49548b1f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 28482 INFO  (Thread-96) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 28486 INFO  (Thread-96) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 28576 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkTestServer start zk server on port:41289
   [junit4]   2> 28697 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 28704 ERROR (Thread-96) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 30676 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 30681 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/schema-inplace-updates.xml to /configs/conf1/schema.xml
   [junit4]   2> 30691 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 30698 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 30700 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 30702 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 30705 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 30707 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 30710 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 30712 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 30730 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 30734 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e9d6a17440001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.base/java.lang.Thread.run(Thread.java:844)
   [junit4]   2> 30817 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 30948 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 30961 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@226881c1{/,null,AVAILABLE}
   [junit4]   2> 30962 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4b4ed55f{SSL,[ssl, http/1.1]}{127.0.0.1:40133}
   [junit4]   2> 30962 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.Server Started @32907ms
   [junit4]   2> 30962 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solr.data.dir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/tempDir-001/control/data, hostPort=40133, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/control-001/cores, replicaType=NRT}
   [junit4]   2> 30963 ERROR (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 30963 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 30963 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 30963 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 30963 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-20T03:52:11.710414Z
   [junit4]   2> 30977 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 30978 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/control-001/solr.xml
   [junit4]   2> 30982 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 31029 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41289/solr
   [junit4]   2> 31514 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 31516 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:40133_
   [junit4]   2> 31518 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.Overseer Overseer (id=98689320831156229-127.0.0.1:40133_-n_0000000000) starting
   [junit4]   2> 31623 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40133_
   [junit4]   2> 31655 INFO  (OverseerStateUpdate-98689320831156229-127.0.0.1:40133_-n_0000000000) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 31800 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 31857 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 31866 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 31869 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/control-001/cores
   [junit4]   2> 32085 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 32086 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41289/solr ready
   [junit4]   2> 32135 INFO  (qtp478678292-351) [    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:40133_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 32180 INFO  (OverseerThreadFactory-272-thread-1) [    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 32342 INFO  (qtp478678292-352) [    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 32342 INFO  (qtp478678292-352) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 32467 INFO  (zkCallback-56-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 33422 INFO  (qtp478678292-352) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 33432 INFO  (qtp478678292-352) [    ] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=inplace-updates
   [junit4]   2> 33435 INFO  (qtp478678292-352) [    ] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
   [junit4]   2> 33435 INFO  (qtp478678292-352) [    ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 33436 INFO  (qtp478678292-352) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 33436 INFO  (qtp478678292-352) [    ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 33436 INFO  (qtp478678292-352) [    ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 33467 WARN  (qtp478678292-352) [    ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 33658 INFO  (qtp478678292-352) [    ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 33658 INFO  (qtp478678292-352) [    ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 33660 INFO  (qtp478678292-352) [    ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 33660 INFO  (qtp478678292-352) [    ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 33666 INFO  (qtp478678292-352) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@22f7568c[control_collection_shard1_replica_n1] main]
   [junit4]   2> 33671 INFO  (qtp478678292-352) [    ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 33672 INFO  (qtp478678292-352) [    ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 33672 INFO  (qtp478678292-352) [    ] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 33674 INFO  (qtp478678292-352) [    ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579029138685034496
   [junit4]   2> 33694 INFO  (searcherExecutor-275-thread-1) [    ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@22f7568c[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 33698 INFO  (qtp478678292-352) [    ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 33698 INFO  (qtp478678292-352) [    ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 33698 INFO  (qtp478678292-352) [    ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:40133/control_collection_shard1_replica_n1/
   [junit4]   2> 33699 INFO  (qtp478678292-352) [    ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 33699 INFO  (qtp478678292-352) [    ] o.a.s.c.SyncStrategy https://127.0.0.1:40133/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 33699 INFO  (qtp478678292-352) [    ] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 33729 INFO  (qtp478678292-352) [    ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:40133/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 33830 INFO  (zkCallback-56-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 33880 INFO  (qtp478678292-352) [    ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 33886 INFO  (qtp478678292-352) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1560
   [junit4]   2> 33912 INFO  (qtp478678292-351) [    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 33990 INFO  (zkCallback-56-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 34184 INFO  (OverseerCollectionConfigSetProcessor-98689320831156229-127.0.0.1:40133_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 34913 INFO  (qtp478678292-351) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:40133_&wt=javabin&version=2} status=0 QTime=2778
   [junit4]   2> 34915 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e9d6a17440006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.base/java.lang.Thread.run(Thread.java:844)
   [junit4]   2> 34920 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 34921 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41289/solr ready
   [junit4]   2> 34923 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 34935 INFO  (qtp478678292-351) [    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 34942 INFO  (OverseerThreadFactory-272-thread-2) [    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 34944 WARN  (OverseerThreadFactory-272-thread-2) [    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 35152 INFO  (qtp478678292-351) [    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 35153 INFO  (qtp478678292-351) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=217
   [junit4]   2> 35275 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-1-001 of type TLOG
   [junit4]   2> 35276 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 35290 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@70b616bc{/,null,AVAILABLE}
   [junit4]   2> 35291 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@518527ea{SSL,[ssl, http/1.1]}{127.0.0.1:36749}
   [junit4]   2> 35291 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.Server Started @37235ms
   [junit4]   2> 35291 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/tempDir-001/jetty1, hostPort=36749, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-1-001/cores, replicaType=TLOG}
   [junit4]   2> 35291 ERROR (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 35291 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 35291 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 35291 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 35291 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-20T03:52:16.038957Z
   [junit4]   2> 35302 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 35302 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-1-001/solr.xml
   [junit4]   2> 35305 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 35311 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41289/solr
   [junit4]   2> 35359 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 35360 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 35362 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36749_
   [junit4]   2> 35370 INFO  (zkCallback-63-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 35371 INFO  (zkCallback-56-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 35372 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 35448 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 35462 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 35462 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 35464 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-1-001/cores
   [junit4]   2> 35512 INFO  (qtp478678292-354) [    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:36749_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 35528 INFO  (OverseerCollectionConfigSetProcessor-98689320831156229-127.0.0.1:40133_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 35530 INFO  (OverseerThreadFactory-272-thread-3) [    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:36749_ for creating new replica
   [junit4]   2> 35563 INFO  (qtp1429027747-398) [    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 35563 INFO  (qtp1429027747-398) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 35669 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 36586 INFO  (qtp1429027747-398) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 36604 INFO  (qtp1429027747-398) [    ] o.a.s.s.IndexSchema [collection1_shard1_replica_t21] Schema name=inplace-updates
   [junit4]   2> 36606 INFO  (qtp1429027747-398) [    ] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
   [junit4]   2> 36606 INFO  (qtp1429027747-398) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t21' using configuration from collection collection1, trusted=true
   [junit4]   2> 36616 INFO  (qtp1429027747-398) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t21' (registry 'solr.core.collection1.shard1.replica_t21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 36616 INFO  (qtp1429027747-398) [    ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 36616 INFO  (qtp1429027747-398) [    ] o.a.s.c.SolrCore [[collection1_shard1_replica_t21] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-1-001/cores/collection1_shard1_replica_t21], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-1-001/cores/collection1_shard1_replica_t21/data/]
   [junit4]   2> 36630 WARN  (qtp1429027747-398) [    ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 36838 INFO  (qtp1429027747-398) [    ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 36838 INFO  (qtp1429027747-398) [    ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 36840 INFO  (qtp1429027747-398) [    ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 36840 INFO  (qtp1429027747-398) [    ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 36843 INFO  (qtp1429027747-398) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@70cbb66c[collection1_shard1_replica_t21] main]
   [junit4]   2> 36851 INFO  (qtp1429027747-398) [    ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 36852 INFO  (qtp1429027747-398) [    ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 36853 INFO  (qtp1429027747-398) [    ] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 36853 INFO  (qtp1429027747-398) [    ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579029142018457600
   [junit4]   2> 36871 INFO  (qtp1429027747-398) [    ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 36871 INFO  (qtp1429027747-398) [    ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 36871 INFO  (qtp1429027747-398) [    ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:36749/collection1_shard1_replica_t21/
   [junit4]   2> 36871 INFO  (qtp1429027747-398) [    ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 36871 INFO  (qtp1429027747-398) [    ] o.a.s.c.SyncStrategy https://127.0.0.1:36749/collection1_shard1_replica_t21/ has no replicas
   [junit4]   2> 36871 INFO  (qtp1429027747-398) [    ] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 36872 INFO  (qtp1429027747-398) [    ] o.a.s.c.ZkController collection1_shard1_replica_t21 stopping background replication from leader
   [junit4]   2> 36877 INFO  (qtp1429027747-398) [    ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:36749/collection1_shard1_replica_t21/ shard1
   [junit4]   2> 36879 INFO  (searcherExecutor-286-thread-1) [    ] o.a.s.c.SolrCore [collection1_shard1_replica_t21] Registered new searcher Searcher@70cbb66c[collection1_shard1_replica_t21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 36982 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 36993 INFO  (qtp1429027747-398) [    ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 36995 INFO  (qtp1429027747-398) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1431
   [junit4]   2> 37008 INFO  (qtp478678292-354) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:36749_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1496
   [junit4]   2> 37112 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 37135 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-2-001 of type TLOG
   [junit4]   2> 37137 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 37238 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@67ac8287{/,null,AVAILABLE}
   [junit4]   2> 37238 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@77fef7cc{SSL,[ssl, http/1.1]}{127.0.0.1:38277}
   [junit4]   2> 37238 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.Server Started @39183ms
   [junit4]   2> 37239 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/tempDir-001/jetty2, hostPort=38277, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-2-001/cores, replicaType=TLOG}
   [junit4]   2> 37239 ERROR (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 37239 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 37239 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 37239 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 37239 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-20T03:52:17.986916Z
   [junit4]   2> 37268 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 37268 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-2-001/solr.xml
   [junit4]   2> 37272 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 37290 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41289/solr
   [junit4]   2> 37375 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 37376 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 37385 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38277_
   [junit4]   2> 37394 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 37394 INFO  (zkCallback-63-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 37402 INFO  (zkCallback-56-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 37403 INFO  (zkCallback-74-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 37535 INFO  (OverseerCollectionConfigSetProcessor-98689320831156229-127.0.0.1:40133_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 37656 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 37719 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 37719 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 37725 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-2-001/cores
   [junit4]   2> 37840 INFO  (qtp78434337-421) [    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38277_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 37851 INFO  (OverseerThreadFactory-272-thread-4) [    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:38277_ for creating new replica
   [junit4]   2> 37877 INFO  (qtp78434337-427) [    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 37877 INFO  (qtp78434337-427) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 38018 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 38018 INFO  (zkCallback-74-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 38915 INFO  (qtp78434337-427) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 38922 INFO  (qtp78434337-427) [    ] o.a.s.s.IndexSchema [collection1_shard1_replica_t23] Schema name=inplace-updates
   [junit4]   2> 38924 INFO  (qtp78434337-427) [    ] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
   [junit4]   2> 38924 INFO  (qtp78434337-427) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t23' using configuration from collection collection1, trusted=true
   [junit4]   2> 38924 INFO  (qtp78434337-427) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t23' (registry 'solr.core.collection1.shard1.replica_t23') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 38925 INFO  (qtp78434337-427) [    ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 38925 INFO  (qtp78434337-427) [    ] o.a.s.c.SolrCore [[collection1_shard1_replica_t23] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-2-001/cores/collection1_shard1_replica_t23], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-2-001/cores/collection1_shard1_replica_t23/data/]
   [junit4]   2> 38928 WARN  (qtp78434337-427) [    ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 38967 INFO  (qtp78434337-427) [    ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 38967 INFO  (qtp78434337-427) [    ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 38969 INFO  (qtp78434337-427) [    ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 38969 INFO  (qtp78434337-427) [    ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 38970 INFO  (qtp78434337-427) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1efad187[collection1_shard1_replica_t23] main]
   [junit4]   2> 38971 INFO  (qtp78434337-427) [    ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 38971 INFO  (qtp78434337-427) [    ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 38972 INFO  (qtp78434337-427) [    ] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 38973 INFO  (searcherExecutor-297-thread-1) [    ] o.a.s.c.SolrCore [collection1_shard1_replica_t23] Registered new searcher Searcher@1efad187[collection1_shard1_replica_t23] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 38973 INFO  (qtp78434337-427) [    ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579029144241438720
   [junit4]   2> 38976 INFO  (qtp78434337-427) [    ] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_t23
   [junit4]   2> 38982 INFO  (updateExecutor-71-thread-1) [    ] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 38982 INFO  (qtp78434337-427) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1105
   [junit4]   2> 38985 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 38985 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 38985 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.ZkController collection1_shard1_replica_t23 stopping background replication from leader
   [junit4]   2> 38985 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_t23]
   [junit4]   2> 38985 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 38985 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_t23] as recovering, leader is [https://127.0.0.1:36749/collection1_shard1_replica_t21/] and I am [https://127.0.0.1:38277/collection1_shard1_replica_t23/]
   [junit4]   2> 38986 INFO  (qtp78434337-421) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:38277_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1146
   [junit4]   2> 38993 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:36749]; [WaitForState: action=PREPRECOVERY&core=collection1_shard1_replica_t21&nodeName=127.0.0.1:38277_&coreNodeName=core_node24&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 38998 INFO  (qtp1429027747-397) [    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
   [junit4]   2> 38998 INFO  (qtp1429027747-397) [    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_t21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:38277_, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_t23","base_url":"https://127.0.0.1:38277","node_name":"127.0.0.1:38277_","state":"down","type":"TLOG"}
   [junit4]   2> 39054 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-3-001 of type TLOG
   [junit4]   2> 39055 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 39056 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@58b2fa14{/,null,AVAILABLE}
   [junit4]   2> 39061 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@69ce12df{SSL,[ssl, http/1.1]}{127.0.0.1:35301}
   [junit4]   2> 39061 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.Server Started @41005ms
   [junit4]   2> 39061 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/tempDir-001/jetty3, hostPort=35301, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-3-001/cores, replicaType=TLOG}
   [junit4]   2> 39061 ERROR (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 39062 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 39062 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 39062 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 39062 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-20T03:52:19.809329Z
   [junit4]   2> 39067 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 39067 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-3-001/solr.xml
   [junit4]   2> 39071 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 39073 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41289/solr
   [junit4]   2> 39080 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 39086 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 39087 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35301_
   [junit4]   2> 39088 INFO  (zkCallback-63-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 39088 INFO  (zkCallback-74-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 39088 INFO  (zkCallback-56-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 39088 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 39090 INFO  (zkCallback-81-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 39144 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 39156 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 39156 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 39158 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-3-001/cores
   [junit4]   2> 39189 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 39189 INFO  (zkCallback-74-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 39252 INFO  (qtp1502513367-461) [    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35301_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 39255 INFO  (OverseerCollectionConfigSetProcessor-98689320831156229-127.0.0.1:40133_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 39256 INFO  (OverseerThreadFactory-272-thread-5) [    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:35301_ for creating new replica
   [junit4]   2> 39267 INFO  (qtp1502513367-455) [    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t25&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 39267 INFO  (qtp1502513367-455) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 39375 INFO  (zkCallback-81-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 39375 INFO  (zkCallback-74-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 39375 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 39999 INFO  (qtp1429027747-397) [    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_t21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:38277_, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_t23","base_url":"https://127.0.0.1:38277","node_name":"127.0.0.1:38277_","state":"recovering","type":"TLOG"}
   [junit4]   2> 39999 INFO  (qtp1429027747-397) [    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 39999 INFO  (qtp1429027747-397) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:38277_&onlyIfLeaderActive=true&core=collection1_shard1_replica_t21&coreNodeName=core_node24&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1001
   [junit4]   2> 40282 INFO  (qtp1502513367-455) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 40288 INFO  (qtp1502513367-455) [    ] o.a.s.s.IndexSchema [collection1_shard1_replica_t25] Schema name=inplace-updates
   [junit4]   2> 40290 INFO  (qtp1502513367-455) [    ] o.a.s.s.IndexSchema Loaded schema inplace-updates/1.6 with uniqueid field id
   [junit4]   2> 40290 INFO  (qtp1502513367-455) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t25' using configuration from collection collection1, trusted=true
   [junit4]   2> 40290 INFO  (qtp1502513367-455) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t25' (registry 'solr.core.collection1.shard1.replica_t25') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@6938493b
   [junit4]   2> 40291 INFO  (qtp1502513367-455) [    ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 40291 INFO  (qtp1502513367-455) [    ] o.a.s.c.SolrCore [[collection1_shard1_replica_t25] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-3-001/cores/collection1_shard1_replica_t25], dataDir=[/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001/shard-3-001/cores/collection1_shard1_replica_t25/data/]
   [junit4]   2> 40295 WARN  (qtp1502513367-455) [    ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 40327 INFO  (qtp1502513367-455) [    ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 40328 INFO  (qtp1502513367-455) [    ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 40329 INFO  (qtp1502513367-455) [    ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 40329 INFO  (qtp1502513367-455) [    ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 40330 INFO  (qtp1502513367-455) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@253dec80[collection1_shard1_replica_t25] main]
   [junit4]   2> 40331 INFO  (qtp1502513367-455) [    ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 40331 INFO  (qtp1502513367-455) [    ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 40331 INFO  (qtp1502513367-455) [    ] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 40333 INFO  (searcherExecutor-308-thread-1) [    ] o.a.s.c.SolrCore [collection1_shard1_replica_t25] Registered new searcher Searcher@253dec80[collection1_shard1_replica_t25] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 40333 INFO  (qtp1502513367-455) [    ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579029145667502080
   [junit4]   2> 40336 INFO  (qtp1502513367-455) [    ] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_t25
   [junit4]   2> 40336 INFO  (updateExecutor-78-thread-1) [    ] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 40337 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 40337 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 40337 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.ZkController collection1_shard1_replica_t25 stopping background replication from leader
   [junit4]   2> 40337 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_t25]
   [junit4]   2> 40337 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 40337 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_t25] as recovering, leader is [https://127.0.0.1:36749/collection1_shard1_replica_t21/] and I am [https://127.0.0.1:35301/collection1_shard1_replica_t25/]
   [junit4]   2> 40337 INFO  (qtp1502513367-455) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t25&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1071
   [junit4]   2> 40341 INFO  (qtp1502513367-461) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:35301_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1089
   [junit4]   2> 40345 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:36749]; [WaitForState: action=PREPRECOVERY&core=collection1_shard1_replica_t21&nodeName=127.0.0.1:35301_&coreNodeName=core_node26&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 40358 INFO  (qtp1429027747-399) [    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node26, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
   [junit4]   2> 40359 INFO  (qtp1429027747-399) [    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_t21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:35301_, coreNodeName=core_node26, onlyIfActiveCheckResult=false, nodeProps: core_node26:{"core":"collection1_shard1_replica_t25","base_url":"https://127.0.0.1:35301","node_name":"127.0.0.1:35301_","state":"down","type":"TLOG"}
   [junit4]   2> 40359 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 40359 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):330
   [junit4]   1> -
   [junit4]   1> replica:core_node22 rstate:active live:true
   [junit4]   1> replica:core_node24 rstate:recovering live:true
   [junit4]   1> replica:core_node26 rstate:down live:true
   [junit4]   2> 40442 INFO  (zkCallback-81-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 40442 INFO  (zkCallback-74-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 40442 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 40501 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
   [junit4]   2> 40501 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Attempting to replicate from [https://127.0.0.1:36749/collection1_shard1_replica_t21/].
   [junit4]   2> 40516 INFO  (qtp1429027747-398) [    ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1579029145859391488,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 40516 INFO  (qtp1429027747-398) [    ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 40516 INFO  (qtp1429027747-398) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 40516 INFO  (qtp1429027747-398) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t21]  webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2}{commit=} 0 2
   [junit4]   2> 40530 INFO  (qtp1429027747-392) [    ] o.a.s.c.S.Request [collection1_shard1_replica_t21]  webapp= path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 40533 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 40533 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 40533 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 40533 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 40534 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Replication Recovery was successful.
   [junit4]   2> 40534 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
   [junit4]   2> 40534 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.ZkController collection1_shard1_replica_t23 starting background replication from leader
   [junit4]   2> 40534 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 40536 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 40542 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Updating version bucket highest from index after successful recovery.
   [junit4]   2> 40542 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579029145886654464
   [junit4]   2> 40544 INFO  (recoveryExecutor-72-thread-1) [    ] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[true]
   [junit4]   2> 40644 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 40644 INFO  (zkCallback-81-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 40644 INFO  (zkCallback-74-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41262 INFO  (OverseerCollectionConfigSetProcessor-98689320831156229-127.0.0.1:40133_-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 41359 INFO  (qtp1429027747-399) [    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_t21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:35301_, coreNodeName=core_node26, onlyIfActiveCheckResult=false, nodeProps: core_node26:{"core":"collection1_shard1_replica_t25","base_url":"https://127.0.0.1:35301","node_name":"127.0.0.1:35301_","state":"recovering","type":"TLOG"}
   [junit4]   2> 41359 INFO  (qtp1429027747-399) [    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node26, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 41359 INFO  (qtp1429027747-399) [    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:35301_&onlyIfLeaderActive=true&core=collection1_shard1_replica_t21&coreNodeName=core_node26&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1001
   [junit4]   1> -
   [junit4]   1> replica:core_node22 rstate:active live:true
   [junit4]   1> replica:core_node24 rstate:active live:true
   [junit4]   1> replica:core_node26 rstate:recovering live:true
   [junit4]   2> 41862 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
   [junit4]   2> 41862 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Attempting to replicate from [https://127.0.0.1:36749/collection1_shard1_replica_t21/].
   [junit4]   2> 41869 INFO  (qtp1429027747-397) [    ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1579029147278114816,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 41870 INFO  (qtp1429027747-397) [    ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 41870 INFO  (qtp1429027747-397) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 41870 INFO  (qtp1429027747-397) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t21]  webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2}{commit=} 0 2
   [junit4]   2> 41872 INFO  (indexFetcher-314-thread-1) [    ] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:36749/collection1_shard1_replica_t21/
   [junit4]   2> 41874 INFO  (qtp1429027747-399) [    ] o.a.s.c.S.Request [collection1_shard1_replica_t21]  webapp= path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 41876 INFO  (indexFetcher-314-thread-1) [    ] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 41876 INFO  (indexFetcher-314-thread-1) [    ] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 41876 INFO  (indexFetcher-314-thread-1) [    ] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 41876 INFO  (indexFetcher-314-thread-1) [    ] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 41880 INFO  (qtp1429027747-399) [    ] o.a.s.c.S.Request [collection1_shard1_replica_t21]  webapp= path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 41881 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 41881 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 41881 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 41881 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 41881 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Replication Recovery was successful.
   [junit4]   2> 41881 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
   [junit4]   2> 41881 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.ZkController collection1_shard1_replica_t25 starting background replication from leader
   [junit4]   2> 41881 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 41885 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 41886 INFO  (indexFetcher-316-thread-1) [    ] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:36749/collection1_shard1_replica_t21/
   [junit4]   2> 41890 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Updating version bucket highest from index after successful recovery.
   [junit4]   2> 41890 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1579029147300134912
   [junit4]   2> 41892 INFO  (recoveryExecutor-79-thread-1) [    ] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[true]
   [junit4]   2> 41894 INFO  (qtp1429027747-392) [    ] o.a.s.c.S.Request [collection1_shard1_replica_t21]  webapp= path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 41894 INFO  (indexFetcher-316-thread-1) [    ] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 41895 INFO  (indexFetcher-316-thread-1) [    ] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 41895 INFO  (indexFetcher-316-thread-1) [    ] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 41895 INFO  (indexFetcher-316-thread-1) [    ] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 41993 INFO  (zkCallback-74-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41993 INFO  (zkCallback-68-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 41994 INFO  (zkCallback-81-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   1> -
   [junit4]   1> replica:core_node22 rstate:active live:true
   [junit4]   1> replica:core_node24 rstate:active live:true
   [junit4]   1> replica:core_node26 rstate:active live:true
   [junit4]   1> no one is recoverying
   [junit4]   2> 42363 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
   [junit4]   2> 42369 INFO  (qtp78434337-425) [    ] o.a.s.c.S.Request [collection1_shard1_replica_t23]  webapp= path=/schema/fields/inplace_updatable_int params={wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 42394 INFO  (qtp1429027747-397) [    ] o.a.s.c.S.Request [collection1_shard1_replica_t21]  webapp= path=/schema/fields/inplace_updatable_float params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 42401 INFO  (qtp78434337-425) [    ] o.a.s.c.S.Request [collection1_shard1_replica_t23]  webapp= path=/schema/fields/_version_ params={wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 42401 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.u.TestInPlaceUpdatesDistrib Trying num docs = 38
   [junit4]   2> 42434 INFO  (qtp1502513367-461) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t25]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36749/collection1_shard1_replica_t21/&wt=javabin&version=2}{add=[-1 (1579029147841200128)]} 0 0
   [junit4]   2> 42435 INFO  (qtp78434337-425) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t23]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36749/collection1_shard1_replica_t21/&wt=javabin&version=2}{add=[-1 (1579029147841200128)]} 0 8
   [junit4]   2> 42435 INFO  (qtp1429027747-394) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t21]  webapp= path=/update params={versions=true&wt=javabin&version=2}{add=[-1 (1579029147841200128)]} 0 29
   [junit4]   2> 42451 INFO  (qtp1502513367-460) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t25]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36749/collection1_shard1_replica_t21/&wt=javabin&version=2}{add=[-2 (1579029147873705984)]} 0 12
   [junit4]   2> 42465 INFO  (qtp78434337-421) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t23]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36749/collection1_shard1_replica_t21/&wt=javabin&version=2}{add=[-2 (1579029147873705984)]} 0 26
   [junit4]   2> 42466 INFO  (qtp1429027747-399) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t21]  webapp= path=/update params={versions=true&wt=javabin&version=2}{add=[-2 (1579029147873705984)]} 0 29
   [junit4]   2> 42479 INFO  (qtp1502513367-458) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t25]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36749/collection1_shard1_replica_t21/&wt=javabin&version=2}{add=[0 (1579029147906211840)]} 0 10
   [junit4]   2> 42480 INFO  (qtp78434337-422) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t23]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36749/collection1_shard1_replica_t21/&wt=javabin&version=2}{add=[0 (1579029147906211840)]} 0 11
   [junit4]   2> 42481 INFO  (qtp1429027747-393) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t21]  webapp= path=/update params={versions=true&wt=javabin&version=2}{add=[0 (1579029147906211840)]} 0 13
   [junit4]   2> 42495 INFO  (qtp1502513367-455) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t25]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36749/collection1_shard1_replica_t21/&wt=javabin&version=2}{add=[-3 (1579029147920891904)]} 0 11
   [junit4]   2> 42496 INFO  (qtp78434337-423) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t23]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36749/collection1_shard1_replica_t21/&wt=javabin&version=2}{add=[-3 (1579029147920891904)]} 0 12
   [junit4]   2> 42497 INFO  (qtp1429027747-392) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t21]  webapp= path=/update params={versions=true&wt=javabin&version=2}{add=[-3 (1579029147920891904)]} 0 15
   [junit4]   2> 42517 INFO  (qtp1502513367-461) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t25]  webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:36749/collection1_shard1_replica_t21/&wt=javabin&version=2}{add=[-4 (1579029147937669120)]} 0 15
   [junit4]   2> 42518 INFO  (qtp78434337-425) [    ] o.a.s.u.p.LogUpdateP

[...truncated too long message...]

 105208 INFO  (zkCallback-81-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 105210 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 105210 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5e6fac2a: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5b5adf34
   [junit4]   2> 105214 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 105214 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@428a7722: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@70d400de
   [junit4]   2> 105215 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 105218 INFO  (coreCloseExecutor-352-thread-1) [    ] o.a.s.c.SolrCore [collection1_shard1_replica_t25]  CLOSING SolrCore org.apache.solr.core.SolrCore@379620e
   [junit4]   2> 105218 INFO  (coreCloseExecutor-352-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.replica_t25, tag=58286606
   [junit4]   2> 105218 INFO  (coreCloseExecutor-352-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2567978c: rootName = null, domain = solr.core.collection1.shard1.replica_t25, service url = null, agent id = null] for registry solr.core.collection1.shard1.replica_t25 / com.codahale.metrics.MetricRegistry@3430a29
   [junit4]   2> 105230 INFO  (coreCloseExecutor-352-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=58286606
   [junit4]   2> 105248 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.Overseer Overseer (id=98689320831156240-127.0.0.1:35301_-n_0000000003) closing
   [junit4]   2> 105248 INFO  (OverseerStateUpdate-98689320831156240-127.0.0.1:35301_-n_0000000003) [    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:35301_
   [junit4]   2> 105258 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@69ce12df{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 105258 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@58b2fa14{/,null,UNAVAILABLE}
   [junit4]   2> 105260 ERROR (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 105260 INFO  (TEST-TestInPlaceUpdatesDistrib.test-seed#[9C854BAF3D78B37E]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:41289 41289
   [junit4]   2> 111054 INFO  (Thread-96) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:41289 41289
   [junit4]   2> 111055 WARN  (Thread-96) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	4	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/clusterstate.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	3	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	3	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestInPlaceUpdatesDistrib -Dtests.method=test -Dtests.seed=9C854BAF3D78B37E -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ru-UA -Dtests.timezone=Etc/GMT-3 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 82.6s J2 | TestInPlaceUpdatesDistrib.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: inplace_updatable_float didn't match for replica at client: https://127.0.0.1:38277/collection1 expected:<109.0> but was:<1.0>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9C854BAF3D78B37E:14D174759384DE86]:0)
   [junit4]    > 	at org.apache.solr.update.TestInPlaceUpdatesDistrib.assertReplicaValue(TestInPlaceUpdatesDistrib.java:980)
   [junit4]    > 	at org.apache.solr.update.TestInPlaceUpdatesDistrib.outOfOrderUpdatesIndividualReplicaTest(TestInPlaceUpdatesDistrib.java:643)
   [junit4]    > 	at org.apache.solr.update.TestInPlaceUpdatesDistrib.test(TestInPlaceUpdatesDistrib.java:144)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:844)
   [junit4]   2> 111057 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 111057 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=359606142
   [junit4]   2> 111057 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 111057 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4e0b426: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / null
   [junit4]   2> 111057 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 111057 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5e64cc76: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / null
   [junit4]   2> 111057 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 111057 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@36e1c2e5: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / null
   [junit4]   2> 111057 INFO  (SUITE-TestInPlaceUpdatesDistrib-seed#[9C854BAF3D78B37E]-worker) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 111060 INFO  (coreCloseExecutor-354-thread-1) [    ] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@63fdbbb7
   [junit4]   2> 111060 INFO  (coreCloseExecutor-354-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1677573047
   [junit4]   2> 111060 INFO  (coreCloseExecutor-354-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@91a9591: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / null
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J2/temp/solr.update.TestInPlaceUpdatesDistrib_9C854BAF3D78B37E-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=ru-UA, timezone=Etc/GMT-3
   [junit4]   2> NOTE: Linux 4.10.0-33-generic amd64/Oracle Corporation 9 (64-bit)/cpus=8,threads=1,free=220476696,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestFieldResource, TestJsonFacetRefinement, PolyFieldTest, FastVectorHighlighterTest, EnumFieldTest, BadIndexSchemaTest, PathHierarchyTokenizerFactoryTest, TestLMJelinekMercerSimilarityFactory, URLClassifyProcessorTest, DistributedQueryComponentCustomSortTest, TestQueryUtils, CopyFieldTest, CdcrRequestHandlerTest, CoreSorterTest, TestInPlaceUpdatesDistrib]
   [junit4] Completed [37/733 (1!)] on J2 in 82.95s, 1 test, 1 failure <<< FAILURES!

[...truncated 41456 lines...]

[JENKINS] Lucene-Solr-master-Linux (32bit/jdk1.8.0_144) - Build # 20501 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/20501/
Java: 32bit/jdk1.8.0_144 -client -XX:+UseG1GC

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:     1) Thread[id=8403, name=jetty-launcher-1337-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)    2) Thread[id=8393, name=jetty-launcher-1337-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 
   1) Thread[id=8403, name=jetty-launcher-1337-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   2) Thread[id=8393, name=jetty-launcher-1337-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
        at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
        at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
        at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
        at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
        at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
        at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
        at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
        at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
	at __randomizedtesting.SeedInfo.seed([13B45E8ADAFAC4EC]:0)




Build Log:
[...truncated 12289 lines...]
   [junit4] Suite: org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth
   [junit4]   2> 1101649 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_13B45E8ADAFAC4EC-001/init-core-data-001
   [junit4]   2> 1101650 WARN  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=302 numCloses=302
   [junit4]   2> 1101650 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1101652 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1101692 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_13B45E8ADAFAC4EC-001/tempDir-001
   [junit4]   2> 1101692 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1101692 INFO  (Thread-1880) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1101692 INFO  (Thread-1880) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1101693 ERROR (Thread-1880) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1101792 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:42999
   [junit4]   2> 1101804 INFO  (jetty-launcher-1337-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1101808 INFO  (jetty-launcher-1337-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1947792{/solr,null,AVAILABLE}
   [junit4]   2> 1101808 INFO  (jetty-launcher-1337-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1101810 INFO  (jetty-launcher-1337-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@ee287f{SSL,[ssl, http/1.1]}{127.0.0.1:34801}
   [junit4]   2> 1101810 INFO  (jetty-launcher-1337-thread-1) [    ] o.e.j.s.Server Started @1103026ms
   [junit4]   2> 1101810 INFO  (jetty-launcher-1337-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=34801}
   [junit4]   2> 1101811 ERROR (jetty-launcher-1337-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1101811 INFO  (jetty-launcher-1337-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1101811 INFO  (jetty-launcher-1337-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1101811 INFO  (jetty-launcher-1337-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1101811 INFO  (jetty-launcher-1337-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-19T23:13:58.468Z
   [junit4]   2> 1101816 INFO  (jetty-launcher-1337-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@d62e32{/solr,null,AVAILABLE}
   [junit4]   2> 1101817 INFO  (jetty-launcher-1337-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@11e0ce3{SSL,[ssl, http/1.1]}{127.0.0.1:35019}
   [junit4]   2> 1101817 INFO  (jetty-launcher-1337-thread-2) [    ] o.e.j.s.Server Started @1103032ms
   [junit4]   2> 1101817 INFO  (jetty-launcher-1337-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35019}
   [junit4]   2> 1101817 ERROR (jetty-launcher-1337-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1101817 INFO  (jetty-launcher-1337-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 8.0.0
   [junit4]   2> 1101817 INFO  (jetty-launcher-1337-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1101817 INFO  (jetty-launcher-1337-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1101817 INFO  (jetty-launcher-1337-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-19T23:13:58.474Z
   [junit4]   2> 1101820 INFO  (jetty-launcher-1337-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1101828 INFO  (jetty-launcher-1337-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1101844 INFO  (jetty-launcher-1337-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42999/solr
   [junit4]   2> 1101846 INFO  (jetty-launcher-1337-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42999/solr
   [junit4]   2> 1101961 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1101962 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34801_solr
   [junit4]   2> 1101963 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.c.Overseer Overseer (id=98688226974105605-127.0.0.1:34801_solr-n_0000000000) starting
   [junit4]   2> 1101967 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1101969 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35019_solr
   [junit4]   2> 1101971 INFO  (zkCallback-1348-thread-1-processing-n:127.0.0.1:34801_solr) [n:127.0.0.1:34801_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1101971 INFO  (zkCallback-1349-thread-1-processing-n:127.0.0.1:35019_solr) [n:127.0.0.1:35019_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1101978 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34801_solr
   [junit4]   2> 1101979 INFO  (zkCallback-1349-thread-1-processing-n:127.0.0.1:35019_solr) [n:127.0.0.1:35019_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1101985 INFO  (zkCallback-1348-thread-1-processing-n:127.0.0.1:34801_solr) [n:127.0.0.1:34801_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1102037 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 1102040 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=jenkins, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=localhost.localdomain,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
   [junit4]   2> 1102045 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1102063 INFO  (jetty-launcher-1337-thread-1-EventThread) [n:127.0.0.1:34801_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1102073 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 1102118 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 1102118 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=jenkins, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=localhost.localdomain,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
   [junit4]   2> 1102120 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1102125 INFO  (jetty-launcher-1337-thread-2-EventThread) [n:127.0.0.1:35019_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1102128 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34801.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb4695
   [junit4]   2> 1102137 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 1102140 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34801.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb4695
   [junit4]   2> 1102140 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_34801.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb4695
   [junit4]   2> 1102141 INFO  (jetty-launcher-1337-thread-1) [n:127.0.0.1:34801_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_13B45E8ADAFAC4EC-001/tempDir-001/node1/.
   [junit4]   2> 1102185 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35019.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb4695
   [junit4]   2> 1102195 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35019.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb4695
   [junit4]   2> 1102195 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35019.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bb4695
   [junit4]   2> 1102197 INFO  (jetty-launcher-1337-thread-2) [n:127.0.0.1:35019_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_13B45E8ADAFAC4EC-001/tempDir-001/node2/.
   [junit4]   2> 1102223 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e9c6b684e0009, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1102227 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1102228 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42999/solr ready
   [junit4]   2> 1102257 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
   [junit4]   2> 1102353 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[13B45E8ADAFAC4EC]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=simple-dt&e=1505898839007&s=wlPs7voSB71QMrBxR72yCk48SKg="; Domain=127.0.0.1; Expires=Qua, 20-set-2017 09:13:59 GMT; Secure; HttpOnly". Invalid 'expires' attribute: Qua, 20-set-2017 09:13:59 GMT
   [junit4]   2> 1102354 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
   [junit4]   2> 1102380 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
   [junit4]   2> 1102418 INFO  (qtp17016773-8348) [n:127.0.0.1:34801_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1102419 INFO  (qtp17016773-8348) [n:127.0.0.1:34801_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1102420 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[13B45E8ADAFAC4EC]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=simple-dt&e=1505898839047&s=vUExxX6BvjrFHPCLCz0EnCfijl8="; Domain=127.0.0.1; Expires=Qua, 20-set-2017 09:13:59 GMT; Secure; HttpOnly". Invalid 'expires' attribute: Qua, 20-set-2017 09:13:59 GMT
   [junit4]   2> 1102420 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
   [junit4]   2> 1102453 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
   [junit4]   2> 1102462 INFO  (qtp17016773-8347) [n:127.0.0.1:34801_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1102462 INFO  (qtp17016773-8347) [n:127.0.0.1:34801_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1102463 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[13B45E8ADAFAC4EC]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=simple-dt&e=1505898839117&s=z6Rmcz8dr2QUbKxb+M/S9+SGlGQ="; Domain=127.0.0.1; Expires=Qua, 20-set-2017 09:13:59 GMT; Secure; HttpOnly". Invalid 'expires' attribute: Qua, 20-set-2017 09:13:59 GMT
   [junit4]   2> 1102464 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
   [junit4]   2> 1102495 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
   [junit4]   2> 1102503 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[13B45E8ADAFAC4EC]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=simple-dt&e=1505898839157&s=EUsxUUaka0Fan0xhmFLqgMXZOk0="; Domain=127.0.0.1; Expires=Qua, 20-set-2017 09:13:59 GMT; Secure; HttpOnly". Invalid 'expires' attribute: Qua, 20-set-2017 09:13:59 GMT
   [junit4]   2> 1102503 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
   [junit4]   2> 1102530 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
   [junit4]   2> 1102536 INFO  (qtp17016773-8351) [    ] o.a.s.s.SolrDispatchFilter Error authenticating
   [junit4]   2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
   [junit4]   2> 	at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1102537 WARN  (qtp17016773-8351) [    ] o.e.j.s.ServletHandler /solr/admin/collections
   [junit4]   2> org.apache.solr.common.SolrException: Error during request authentication, 
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
   [junit4]   2> 	at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
   [junit4]   2> 	at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
   [junit4]   2> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
   [junit4]   2> 	at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
   [junit4]   2> 	at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
   [junit4]   2> 	... 29 more
   [junit4]   2> 1102539 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
   [junit4] IGNOR/A 0.00s J1 | TestImpersonationWithHadoopAuth.testForwarding
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/HADOOP-9893))
   [junit4]   2> 1102572 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
   [junit4]   2> 1117600 WARN  (jetty-launcher-1337-thread-2-SendThread(127.0.0.1:42999)) [n:127.0.0.1:35019_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15402ms for sessionid 0x15e9c6b684e0006
   [junit4]   2> 1117600 WARN  (jetty-launcher-1337-thread-1-SendThread(127.0.0.1:42999)) [n:127.0.0.1:34801_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15446ms for sessionid 0x15e9c6b684e0007
   [junit4]   2> 1117600 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e9c6b684e0006, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1117600 WARN  (jetty-launcher-1337-thread-2-SendThread(127.0.0.1:42999)) [n:127.0.0.1:35019_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15445ms for sessionid 0x15e9c6b684e0008
   [junit4]   2> 1117600 WARN  (jetty-launcher-1337-thread-1-SendThread(127.0.0.1:42999)) [n:127.0.0.1:34801_solr    ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15444ms for sessionid 0x15e9c6b684e0005
   [junit4]   2> 1117600 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e9c6b684e0007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1117601 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e9c6b684e0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1117601 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e9c6b684e0008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1117602 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[13B45E8ADAFAC4EC]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=simple-dt&e=1505898839239&s=GP/k86vKpiFvPcAB1nPq67U/nGM="; Domain=127.0.0.1; Expires=Qua, 20-set-2017 09:13:59 GMT; Secure; HttpOnly". Invalid 'expires' attribute: Qua, 20-set-2017 09:13:59 GMT
   [junit4]   2> 1117602 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
   [junit4]   2> 1117623 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
   [junit4]   2> 1117639 WARN  (qtp17016773-8354) [    ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: ‘bar’: no such user
   [junit4]   2> id: ‘bar’: no such user
   [junit4]   2> 
   [junit4]   2> 1117642 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[13B45E8ADAFAC4EC]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=simple-dt&e=1505898854285&s=GLJ3+cViDWmroz58yxi651K2vU0="; Domain=127.0.0.1; Expires=Qua, 20-set-2017 09:14:14 GMT; Secure; HttpOnly". Invalid 'expires' attribute: Qua, 20-set-2017 09:14:14 GMT
   [junit4]   2> 1117642 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
   [junit4]   2> 1117665 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
   [junit4]   2> 1117670 INFO  (qtp17016773-8354) [n:127.0.0.1:34801_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1117670 INFO  (qtp17016773-8354) [n:127.0.0.1:34801_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1117671 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[13B45E8ADAFAC4EC]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=simple-dt&e=1505898854327&s=/eTAMTBC7dgFvM4m1Dy0cIVgsYc="; Domain=127.0.0.1; Expires=Qua, 20-set-2017 09:14:14 GMT; Secure; HttpOnly". Invalid 'expires' attribute: Qua, 20-set-2017 09:14:14 GMT
   [junit4]   2> 1117671 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
   [junit4]   2> 1117694 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
   [junit4]   2> 1117700 WARN  (zkCallback-1349-thread-1-processing-n:127.0.0.1:35019_solr) [n:127.0.0.1:35019_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@93a0cf name: ZooKeeperConnection Watcher:127.0.0.1:42999/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1117700 WARN  (zkCallback-1349-thread-1-processing-n:127.0.0.1:35019_solr) [n:127.0.0.1:35019_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1117700 INFO  (jetty-launcher-1337-thread-1-EventThread) [n:127.0.0.1:34801_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 1117701 INFO  (jetty-launcher-1337-thread-2-EventThread) [n:127.0.0.1:35019_solr    ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
   [junit4]   2> 1117701 WARN  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[13B45E8ADAFAC4EC]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=simple-dt&e=1505898854355&s=h0BxdCQtLUL56AR2pQHMypAHiW0="; Domain=127.0.0.1; Expires=Qua, 20-set-2017 09:14:14 GMT; Secure; HttpOnly". Invalid 'expires' attribute: Qua, 20-set-2017 09:14:14 GMT
   [junit4]   2> 1117703 WARN  (zkCallback-1355-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@180aab7 name: ZooKeeperConnection Watcher:127.0.0.1:42999/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1117703 WARN  (zkCallback-1355-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1117703 WARN  (zkCallback-1348-thread-1-processing-n:127.0.0.1:34801_solr) [n:127.0.0.1:34801_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5b68b3 name: ZooKeeperConnection Watcher:127.0.0.1:42999/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
   [junit4]   2> 1117704 WARN  (zkCallback-1348-thread-1-processing-n:127.0.0.1:34801_solr) [n:127.0.0.1:34801_solr    ] o.a.s.c.c.ConnectionManager zkClient has disconnected
   [junit4]   2> 1117704 INFO  (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[13B45E8ADAFAC4EC]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
   [junit4]   2> 1118136 INFO  (jetty-closer-1338-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@ee287f{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1118137 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=31644790
   [junit4]   2> 1118137 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1118137 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@15bd1a: rootName = solr_34801, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@505bfb
   [junit4]   2> 1118138 INFO  (jetty-closer-1338-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@11e0ce3{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1118138 INFO  (jetty-closer-1338-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=30720045
   [junit4]   2> 1118138 INFO  (jetty-closer-1338-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1118138 INFO  (jetty-closer-1338-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1ab2845: rootName = solr_35019, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@752bef
   [junit4]   2> 1118146 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1118146 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1026396: rootName = solr_34801, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@4541c4
   [junit4]   2> 1118146 INFO  (jetty-closer-1338-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1118147 INFO  (jetty-closer-1338-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@118da83: rootName = solr_35019, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@4541c4
   [junit4]   2> 1118153 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1118153 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@c8eb23: rootName = solr_34801, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@156c935
   [junit4]   2> 1118154 INFO  (jetty-closer-1338-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1118154 INFO  (jetty-closer-1338-thread-2) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@19fecfe: rootName = solr_35019, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@156c935
   [junit4]   2> 1118762 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1118762 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1118762 INFO  (jetty-closer-1338-thread-1) [    ] o.a.s.c.Overseer Overseer (id=98688226974105605-127.0.0.1:34801_solr-n_0000000000) closing
   [junit4]   2> 1118762 INFO  (OverseerStateUpdate-98688226974105605-127.0.0.1:34801_solr-n_0000000000) [n:127.0.0.1:34801_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34801_solr
   [junit4]   2> 1118762 INFO  (OverseerCollectionConfigSetProcessor-98688226974105605-127.0.0.1:34801_solr-n_0000000000) [n:127.0.0.1:34801_solr    ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=98688226974105605-127.0.0.1:34801_solr-n_0000000000) am no longer a leader.
   [junit4]   2> 1118763 WARN  (zkCallback-1348-thread-3-processing-n:127.0.0.1:34801_solr) [n:127.0.0.1:34801_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1118763 ERROR (Thread[Thread-1897,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:34801_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1119084 INFO  (zkCallback-1349-thread-5-processing-n:127.0.0.1:35019_solr) [n:127.0.0.1:35019_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1119084 INFO  (zkCallback-1349-thread-2-processing-n:127.0.0.1:35019_solr) [n:127.0.0.1:35019_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35019_solr
   [junit4]   2> 1119085 INFO  (jetty-closer-1338-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1119085 INFO  (jetty-closer-1338-thread-2) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1119086 WARN  (zkCallback-1349-thread-5-processing-n:127.0.0.1:35019_solr) [n:127.0.0.1:35019_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1119488 INFO  (jetty-closer-1338-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1947792{/solr,null,UNAVAILABLE}
   [junit4]   2> 1120586 ERROR (Thread[Thread-1899,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:35019_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1120587 INFO  (jetty-closer-1338-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@d62e32{/solr,null,UNAVAILABLE}
   [junit4]   2> 1120588 ERROR (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1120588 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42999 42999
   [junit4]   2> 1126369 INFO  (Thread-1880) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42999 42999
   [junit4]   2> 1126371 WARN  (Thread-1880) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 	3	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 1126372 INFO  (SUITE-TestImpersonationWithHadoopAuth-seed#[13B45E8ADAFAC4EC]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Sep 19, 2017 11:14:23 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> Sep 19, 2017 11:14:43 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 
   [junit4]   2>    1) Thread[id=8403, name=jetty-launcher-1337-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2>    2) Thread[id=8393, name=jetty-launcher-1337-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2>         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2>         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2>         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2>         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2>         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2>         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Sep 19, 2017 11:14:43 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=8403, name=jetty-launcher-1337-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2>    2) Thread[id=8393, name=jetty-launcher-1337-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]   2> 1146408 ERROR (jetty-launcher-1337-thread-2-EventThread) [n:127.0.0.1:35019_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> 1146408 ERROR (jetty-launcher-1337-thread-1-EventThread) [n:127.0.0.1:34801_solr    ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]   2> 	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]   2> 	at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]   2> 	at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]   2> 	at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]   2> 	at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]   2> 	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]   2> Sep 19, 2017 11:14:43 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=5, maxDocsPerChunk=8, blockSize=9), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=5, blockSize=9)), sim=RandomSimilarity(queryNorm=false): {}, locale=pt, timezone=Europe/Isle_of_Man
   [junit4]   2> NOTE: Linux 4.10.0-33-generic i386/Oracle Corporation 1.8.0_144 (32-bit)/cpus=8,threads=1,free=137974536,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestRTimerTree, TestSolrCoreParser, TestReloadAndDeleteDocs, ShardRoutingTest, SliceStateTest, LeaderInitiatedRecoveryOnCommitTest, TestFieldTypeResource, MoveReplicaTest, SaslZkACLProviderTest, SolrSlf4jReporterTest, BlobRepositoryCloudTest, SuggestComponentContextFilterQueryTest, FileUtilsTest, PreAnalyzedFieldManagedSchemaCloudTest, TestJsonRequest, TestFieldCacheSortRandom, TestPayloadCheckQParserPlugin, TestBulkSchemaConcurrent, TestRTGBase, DistributedFacetPivotLongTailTest, TestFileDictionaryLookup, ClassificationUpdateProcessorIntegrationTest, TestLegacyFieldCache, BasicFunctionalityTest, TestNamedUpdateProcessors, TestUnifiedSolrHighlighter, TestBackupRepositoryFactory, SolrMetricReporterTest, ShardRoutingCustomTest, ScriptEngineTest, SpellCheckCollatorWithCollapseTest, BadComponentTest, TestGraphMLResponseWriter, TestLegacyFieldReuse, TestClassicSimilarityFactory, TestFastLRUCache, ChaosMonkeySafeLeaderTest, AssignBackwardCompatibilityTest, CdcrReplicationHandlerTest, TestElisionMultitermQuery, CdcrUpdateLogTest, TestFuzzyAnalyzedSuggestions, NoCacheHeaderTest, TestSolrCoreSnapshots, HdfsDirectoryTest, TestNumericTokenStream, TestExpandComponent, TestSimpleTrackingShardHandler, TestStressCloudBlindAtomicUpdates, SolrCmdDistributorTest, HttpSolrCallGetCoreTest, TestRuleBasedAuthorizationPlugin, StatelessScriptUpdateProcessorFactoryTest, AnalysisAfterCoreReloadTest, StatsReloadRaceTest, TestSubQueryTransformerCrossCore, TestDistribDocBasedVersion, CSVRequestHandlerTest, RemoteQueryErrorTest, BigEndianAscendingWordSerializerTest, TestEmbeddedSolrServerConstructors, DistanceFunctionTest, TestFieldCache, TestManagedResourceStorage, CacheHeaderTest, IndexSchemaTest, JsonLoaderTest, TestCollapseQParserPlugin, TestPullReplica, TestManagedSynonymGraphFilterFactory, RankQueryTest, TestStreamBody, TestHashPartitioner, AtomicUpdateProcessorFactoryTest, V2StandaloneTest, TestStressRecovery, TestCoreContainer, TestRandomRequestDistribution, CursorMarkTest, TestBlobHandler, FileBasedSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SolrGraphiteReporterTest, QueryParsingTest, TestDistribIDF, SolrCloudExampleTest, PropertiesRequestHandlerTest, DocValuesMissingTest, LukeRequestHandlerTest, SharedFSAutoReplicaFailoverUtilsTest, TestLeaderInitiatedRecoveryThread, TestSchemaManager, JSONWriterTest, DocValuesMultiTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, DistributedQueueTest, TestSystemIdResolver, ConnectionManagerTest, TestTlogReplica, TestSolrCloudWithDelegationTokens, TestRequestForwarding, TlogReplayBufferedWhileIndexingTest, TestNestedDocsSort, ClusterStateUpdateTest, TestPerFieldSimilarityWithDefaultOverride, TestStressVersions, TestImpersonationWithHadoopAuth]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestImpersonationWithHadoopAuth -Dtests.seed=13B45E8ADAFAC4EC -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=pt -Dtests.timezone=Europe/Isle_of_Man -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestImpersonationWithHadoopAuth (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 
   [junit4]    >    1) Thread[id=8403, name=jetty-launcher-1337-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    >    2) Thread[id=8393, name=jetty-launcher-1337-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
   [junit4]    >         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
   [junit4]    >         at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
   [junit4]    >         at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
   [junit4]    >         at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
   [junit4]    >         at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
   [junit4]    >         at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([13B45E8ADAFAC4EC]:0)
   [junit4] Completed [337/733 (1!)] on J1 in 44.78s, 10 tests, 1 error, 1 skipped <<< FAILURES!

[...truncated 43339 lines...]