You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/10/11 04:48:30 UTC
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 235 -
Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/235/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
3 tests failed.
FAILED: org.apache.solr.core.TestLazyCores.testNoCommit
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([CC480704D68B187D:1328A6D51DAC7BD8]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:884)
at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:847)
at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:829)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int><lst name="params"><str name="q">*:*</str></lst></lst><result name="response" numFound="0" start="0"></result>
</response>
request was:q=*:*
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:877)
... 41 more
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 1) Thread[id=23808, name=searcherExecutor-9002-thread-1, state=WAITING, group=TGRP-TestLazyCores] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:
1) Thread[id=23808, name=searcherExecutor-9002-thread-1, state=WAITING, group=TGRP-TestLazyCores]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([CC480704D68B187D]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.core.TestLazyCores
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=23808, name=searcherExecutor-9002-thread-1, state=WAITING, group=TGRP-TestLazyCores] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=23808, name=searcherExecutor-9002-thread-1, state=WAITING, group=TGRP-TestLazyCores]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([CC480704D68B187D]:0)
Build Log:
[...truncated 12922 lines...]
[junit4] Suite: org.apache.solr.core.TestLazyCores
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/init-core-data-001
[junit4] 2> 2628836 WARN (SUITE-TestLazyCores-seed#[CC480704D68B187D]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=46 numCloses=46
[junit4] 2> 2628836 INFO (SUITE-TestLazyCores-seed#[CC480704D68B187D]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2628837 INFO (SUITE-TestLazyCores-seed#[CC480704D68B187D]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2628837 INFO (SUITE-TestLazyCores-seed#[CC480704D68B187D]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2628839 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRace
[junit4] 2> 2628852 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/solr.xml
[junit4] 2> 2628854 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2628855 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2628855 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2628883 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2628889 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2628889 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2628890 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2628900 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2628900 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2628905 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 2628905 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2628907 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2628907 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/./collection2, trusted=true
[junit4] 2> 2628907 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection2' (registry 'solr.core.collection2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2628913 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2628913 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/./collection2/data/]
[junit4] 2> 2628915 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2628905 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 2628917 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 2628918 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2628918 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/./collection5, trusted=true
[junit4] 2> 2628918 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection5' (registry 'solr.core.collection5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2628918 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2628918 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/./collection5/data/]
[junit4] 2> 2628920 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2628927 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2628927 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/./collection1, trusted=true
[junit4] 2> 2628927 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2628927 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2628927 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/./collection1/data/]
[junit4] 2> 2628934 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2628962 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2628962 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2628963 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2628967 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@74b014c0[collection5] main]
[junit4] 2> 2628967 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/collection5/conf
[junit4] 2> 2628968 INFO (coreLoadExecutor-8880-thread-3) [ x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2628968 INFO (searcherExecutor-8882-thread-1-processing-x:collection5) [ x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@74b014c0[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2628984 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2628984 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2628984 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2628985 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@6ab5bb44[collection2] main]
[junit4] 2> 2628985 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/collection2/conf
[junit4] 2> 2628986 INFO (coreLoadExecutor-8880-thread-2) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2628986 INFO (searcherExecutor-8881-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@6ab5bb44[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2628996 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2628996 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2628997 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2628997 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@73ada7[collection1] main]
[junit4] 2> 2628997 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/collection1/conf
[junit4] 2> 2628998 INFO (coreLoadExecutor-8880-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2628998 INFO (searcherExecutor-8883-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@73ada7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629007 INFO (Thread-4445) [ x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629011 INFO (Thread-4445) [ x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
[junit4] 2> 2629013 INFO (Thread-4445) [ x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629013 INFO (Thread-4445) [ x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/./collection3, trusted=true
[junit4] 2> 2629013 INFO (Thread-4445) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection3' (registry 'solr.core.collection3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629013 INFO (Thread-4445) [ x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629013 INFO (Thread-4445) [ x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/./collection3/data/]
[junit4] 2> 2629015 INFO (Thread-4445) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2629075 INFO (Thread-4445) [ x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629075 INFO (Thread-4445) [ x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629075 INFO (Thread-4445) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2629075 INFO (Thread-4445) [ x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@2f6d61a4[collection3] main]
[junit4] 2> 2629076 INFO (Thread-4445) [ x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-001/collection3/conf
[junit4] 2> 2629076 INFO (Thread-4445) [ x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629080 INFO (searcherExecutor-8893-thread-1-processing-x:collection3) [ x:collection3] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@2f6d61a4[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629082 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2078669499
[junit4] 2> 2629082 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 2629082 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@766de07c: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@59ec3547
[junit4] 2> 2629085 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 2629085 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2e7521c5: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@77bf2a18
[junit4] 2> 2629088 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 2629088 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@16b86b1d: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6a505c1c
[junit4] 2> 2629088 INFO (coreCloseExecutor-8897-thread-1) [ x:collection5] o.a.s.c.SolrCore [collection5] CLOSING SolrCore org.apache.solr.core.SolrCore@334f525e
[junit4] 2> 2629088 INFO (coreCloseExecutor-8897-thread-2) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@38977cc1
[junit4] 2> 2629088 INFO (coreCloseExecutor-8897-thread-3) [ x:collection2] o.a.s.c.SolrCore [collection2] CLOSING SolrCore org.apache.solr.core.SolrCore@75fafc63
[junit4] 2> 2629088 INFO (coreCloseExecutor-8897-thread-4) [ x:collection3] o.a.s.c.SolrCore [collection3] CLOSING SolrCore org.apache.solr.core.SolrCore@af41cf5
[junit4] 2> 2629088 INFO (coreCloseExecutor-8897-thread-1) [ x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=860836446
[junit4] 2> 2629088 INFO (coreCloseExecutor-8897-thread-1) [ x:collection5] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@67f9a9f5: rootName = null, domain = solr.core.collection5, service url = null, agent id = null] for registry solr.core.collection5 / com.codahale.metrics.MetricRegistry@569f9a8b
[junit4] 2> 2629092 INFO (coreCloseExecutor-8897-thread-2) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=949451969
[junit4] 2> 2629092 INFO (coreCloseExecutor-8897-thread-2) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4bf4de51: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@17839dcf
[junit4] 2> 2629097 INFO (coreCloseExecutor-8897-thread-3) [ x:collection2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=1979382883
[junit4] 2> 2629097 INFO (coreCloseExecutor-8897-thread-3) [ x:collection2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3adc5a20: rootName = null, domain = solr.core.collection2, service url = null, agent id = null] for registry solr.core.collection2 / com.codahale.metrics.MetricRegistry@1896f320
[junit4] 2> 2629101 INFO (coreCloseExecutor-8897-thread-4) [ x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection3, tag=183770357
[junit4] 2> 2629101 INFO (coreCloseExecutor-8897-thread-4) [ x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3ca5c59b: rootName = null, domain = solr.core.collection3, service url = null, agent id = null] for registry solr.core.collection3 / com.codahale.metrics.MetricRegistry@592ae19c
[junit4] 2> 2629107 INFO (TEST-TestLazyCores.testRace-seed#[CC480704D68B187D]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testRace
[junit4] 2> 2629109 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCachingLimit
[junit4] 2> 2629121 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/solr.xml
[junit4] 2> 2629123 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2629123 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2629124 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2629149 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629155 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629155 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629156 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2629165 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629165 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629165 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629178 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 2629178 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 2629178 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 2629180 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629180 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629180 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection2, trusted=true
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection5, trusted=true
[junit4] 2> 2629180 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection1, trusted=true
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection2' (registry 'solr.core.collection2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection5' (registry 'solr.core.collection5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection2/data/]
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection5/data/]
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629181 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection1/data/]
[junit4] 2> 2629182 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2629183 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2629182 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2629230 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629230 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629231 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629231 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629231 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629231 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629231 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2629231 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2629231 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2629232 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@d65c547[collection2] main]
[junit4] 2> 2629232 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@1bb6d55d[collection5] main]
[junit4] 2> 2629232 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@50403fd6[collection1] main]
[junit4] 2> 2629232 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection2/conf
[junit4] 2> 2629232 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection5/conf
[junit4] 2> 2629232 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection1/conf
[junit4] 2> 2629232 INFO (coreLoadExecutor-8904-thread-2) [ x:collection2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629232 INFO (coreLoadExecutor-8904-thread-3) [ x:collection5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629232 INFO (coreLoadExecutor-8904-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629233 INFO (searcherExecutor-8905-thread-1-processing-x:collection2) [ x:collection2] o.a.s.c.SolrCore [collection2] Registered new searcher Searcher@d65c547[collection2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629233 INFO (searcherExecutor-8907-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@50403fd6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629233 INFO (searcherExecutor-8906-thread-1-processing-x:collection5) [ x:collection5] o.a.s.c.SolrCore [collection5] Registered new searcher Searcher@1bb6d55d[collection5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629240 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629250 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema [collection3] Schema name=tiny
[junit4] 2> 2629252 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629252 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection3' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection3, trusted=true
[junit4] 2> 2629252 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection3' (registry 'solr.core.collection3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629252 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629252 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore [[collection3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection3/data/]
[junit4] 2> 2629254 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2629289 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629289 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629290 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=57.873046875, floorSegmentMB=1.779296875, forceMergeDeletesPctAllowed=8.38753238070078, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.22217626184762704
[junit4] 2> 2629290 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@6b0e3fc4[collection3] main]
[junit4] 2> 2629290 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection3/conf
[junit4] 2> 2629291 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629291 INFO (searcherExecutor-8917-thread-1-processing-x:collection3) [ x:collection3] o.a.s.c.SolrCore [collection3] Registered new searcher Searcher@6b0e3fc4[collection3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629297 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629301 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema [collection4] Schema name=tiny
[junit4] 2> 2629303 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629303 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection4' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection4, trusted=true
[junit4] 2> 2629303 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection4' (registry 'solr.core.collection4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629303 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629303 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore [[collection4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection4/data/]
[junit4] 2> 2629305 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.625, noCFSRatio=1.0]
[junit4] 2> 2629341 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629341 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629342 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 2629343 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a9f1f36[collection4] main]
[junit4] 2> 2629343 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection4/conf
[junit4] 2> 2629344 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629346 INFO (searcherExecutor-8921-thread-1-processing-x:collection3) [ x:collection3] o.a.s.c.SolrCore [collection4] Registered new searcher Searcher@6a9f1f36[collection4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629354 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629358 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema [collection6] Schema name=tiny
[junit4] 2> 2629360 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629360 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection6' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection6, trusted=true
[junit4] 2> 2629360 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection6' (registry 'solr.core.collection6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629360 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629361 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore [[collection6] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection6], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection6/data/]
[junit4] 2> 2629362 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=18.3525390625, floorSegmentMB=0.4619140625, forceMergeDeletesPctAllowed=0.8857596460002592, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2629408 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629408 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629408 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=79.5625, floorSegmentMB=1.87109375, forceMergeDeletesPctAllowed=11.622052910682488, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2629409 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@65f970fc[collection6] main]
[junit4] 2> 2629409 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection6/conf
[junit4] 2> 2629409 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629410 INFO (searcherExecutor-8925-thread-1-processing-x:collection3) [ x:collection3] o.a.s.c.SolrCore [collection6] Registered new searcher Searcher@65f970fc[collection6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629416 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629419 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema [collection7] Schema name=tiny
[junit4] 2> 2629421 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629421 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection7' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection7, trusted=true
[junit4] 2> 2629421 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection7' (registry 'solr.core.collection7') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629421 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629421 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore [[collection7] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection7], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection7/data/]
[junit4] 2> 2629423 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 2629458 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629459 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629459 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1742849382, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2629460 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a3acb3b[collection7] main]
[junit4] 2> 2629460 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection7/conf
[junit4] 2> 2629460 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629461 INFO (searcherExecutor-8929-thread-1-processing-x:collection3) [ x:collection3] o.a.s.c.SolrCore [collection7] Registered new searcher Searcher@6a3acb3b[collection7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629467 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629471 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema [collection8] Schema name=tiny
[junit4] 2> 2629472 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629472 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection8' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection8, trusted=true
[junit4] 2> 2629473 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection8' (registry 'solr.core.collection8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629473 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629473 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore [[collection8] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection8], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection8/data/]
[junit4] 2> 2629475 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=20.0791015625, floorSegmentMB=1.2802734375, forceMergeDeletesPctAllowed=8.426584743203792, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2629510 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629510 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629511 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=23, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1699932245787273]
[junit4] 2> 2629511 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@4d3d3692[collection8] main]
[junit4] 2> 2629511 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection8/conf
[junit4] 2> 2629512 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629512 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.TransientSolrCoreCacheDefault Closing transient core [collection3]
[junit4] 2> 2629512 INFO (searcherExecutor-8933-thread-1-processing-x:collection3) [ x:collection3] o.a.s.c.SolrCore [collection8] Registered new searcher Searcher@4d3d3692[collection8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629527 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629531 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema [collection9] Schema name=tiny
[junit4] 2> 2629532 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629533 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.CoreContainer Creating SolrCore 'collection9' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection9, trusted=true
[junit4] 2> 2629533 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection9' (registry 'solr.core.collection9') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629533 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629533 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore [[collection9] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection9], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/./collection9/data/]
[junit4] 2> 2629535 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=21, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2629571 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629571 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629572 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=15.0048828125, floorSegmentMB=1.837890625, forceMergeDeletesPctAllowed=17.644635834295272, segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24172479888311627
[junit4] 2> 2629572 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.s.SolrIndexSearcher Opening [Searcher@27b56e36[collection9] main]
[junit4] 2> 2629572 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-002/collection9/conf
[junit4] 2> 2629573 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2629573 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.TransientSolrCoreCacheDefault Closing transient core [collection2]
[junit4] 2> 2629573 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrCore [collection3] CLOSING SolrCore org.apache.solr.core.SolrCore@276e4e5e
[junit4] 2> 2629573 INFO (Thread-4460) [ ] o.a.s.c.SolrCore [collection2] CLOSING SolrCore org.apache.solr.core.SolrCore@2627dc18
[junit4] 2> 2629573 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection3, tag=661540446
[junit4] 2> 2629573 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@33969a64: rootName = null, domain = solr.core.collection3, service url = null, agent id = null] for registry solr.core.collection3 / com.codahale.metrics.MetricRegistry@174e2620
[junit4] 2> 2629575 INFO (searcherExecutor-8937-thread-1-processing-x:collection3) [ x:collection3] o.a.s.c.SolrCore [collection9] Registered new searcher Searcher@27b56e36[collection9] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2629579 INFO (Thread-4460) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection2, tag=640146456
[junit4] 2> 2629580 INFO (Thread-4460) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6e629de6: rootName = null, domain = solr.core.collection2, service url = null, agent id = null] for registry solr.core.collection2 / com.codahale.metrics.MetricRegistry@55c0910b
[junit4] 2> 2629580 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1917613729
[junit4] 2> 2629583 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 2629583 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@410d1249: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@4672508f
[junit4] 2> 2629586 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 2629587 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@d73335e: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@77bf2a18
[junit4] 2> 2629589 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 2629589 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@10a5d5a2: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6a505c1c
[junit4] 2> 2629589 INFO (coreCloseExecutor-8941-thread-1-processing-x:collection3) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@511cf82f
[junit4] 2> 2629589 INFO (coreCloseExecutor-8941-thread-2-processing-x:collection3) [ x:collection5] o.a.s.c.SolrCore [collection5] CLOSING SolrCore org.apache.solr.core.SolrCore@5f33b616
[junit4] 2> 2629589 INFO (coreCloseExecutor-8941-thread-1-processing-x:collection3) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=1360853039
[junit4] 2> 2629589 INFO (coreCloseExecutor-8941-thread-1-processing-x:collection3) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@18dac444: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@3261ad2b
[junit4] 2> 2629589 INFO (coreCloseExecutor-8941-thread-3-processing-x:collection3) [ x:collection4] o.a.s.c.SolrCore [collection4] CLOSING SolrCore org.apache.solr.core.SolrCore@170cc588
[junit4] 2> 2629589 INFO (coreCloseExecutor-8941-thread-6-processing-x:collection3) [ x:collection8] o.a.s.c.SolrCore [collection8] CLOSING SolrCore org.apache.solr.core.SolrCore@256c77e2
[junit4] 2> 2629589 INFO (coreCloseExecutor-8941-thread-7-processing-x:collection3) [ x:collection9] o.a.s.c.SolrCore [collection9] CLOSING SolrCore org.apache.solr.core.SolrCore@24d220cd
[junit4] 2> 2629590 INFO (coreCloseExecutor-8941-thread-5-processing-x:collection3) [ x:collection7] o.a.s.c.SolrCore [collection7] CLOSING SolrCore org.apache.solr.core.SolrCore@3ebfa88e
[junit4] 2> 2629590 INFO (coreCloseExecutor-8941-thread-4-processing-x:collection3) [ x:collection6] o.a.s.c.SolrCore [collection6] CLOSING SolrCore org.apache.solr.core.SolrCore@773ce8ae
[junit4] 2> 2629593 INFO (coreCloseExecutor-8941-thread-2-processing-x:collection3) [ x:collection5] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection5, tag=1597224470
[junit4] 2> 2629593 INFO (coreCloseExecutor-8941-thread-2-processing-x:collection3) [ x:collection5] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@76558fcc: rootName = null, domain = solr.core.collection5, service url = null, agent id = null] for registry solr.core.collection5 / com.codahale.metrics.MetricRegistry@54265e55
[junit4] 2> 2629597 INFO (coreCloseExecutor-8941-thread-3-processing-x:collection3) [ x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection4, tag=386712968
[junit4] 2> 2629597 INFO (coreCloseExecutor-8941-thread-3-processing-x:collection3) [ x:collection4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6b8a027: rootName = null, domain = solr.core.collection4, service url = null, agent id = null] for registry solr.core.collection4 / com.codahale.metrics.MetricRegistry@788e8a17
[junit4] 2> 2629602 INFO (coreCloseExecutor-8941-thread-6-processing-x:collection3) [ x:collection8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection8, tag=627865570
[junit4] 2> 2629602 INFO (coreCloseExecutor-8941-thread-6-processing-x:collection3) [ x:collection8] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@69d3251e: rootName = null, domain = solr.core.collection8, service url = null, agent id = null] for registry solr.core.collection8 / com.codahale.metrics.MetricRegistry@2b2b1ead
[junit4] 2> 2629606 INFO (coreCloseExecutor-8941-thread-7-processing-x:collection3) [ x:collection9] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection9, tag=617750733
[junit4] 2> 2629606 INFO (coreCloseExecutor-8941-thread-7-processing-x:collection3) [ x:collection9] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2e6abe58: rootName = null, domain = solr.core.collection9, service url = null, agent id = null] for registry solr.core.collection9 / com.codahale.metrics.MetricRegistry@3e09c0a9
[junit4] 2> 2629611 INFO (coreCloseExecutor-8941-thread-5-processing-x:collection3) [ x:collection7] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection7, tag=1052747918
[junit4] 2> 2629611 INFO (coreCloseExecutor-8941-thread-5-processing-x:collection3) [ x:collection7] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6aa7a7ce: rootName = null, domain = solr.core.collection7, service url = null, agent id = null] for registry solr.core.collection7 / com.codahale.metrics.MetricRegistry@61f4b15b
[junit4] 2> 2629616 INFO (coreCloseExecutor-8941-thread-4-processing-x:collection3) [ x:collection6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection6, tag=2000480430
[junit4] 2> 2629616 INFO (coreCloseExecutor-8941-thread-4-processing-x:collection3) [ x:collection6] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@77fd7291: rootName = null, domain = solr.core.collection6, service url = null, agent id = null] for registry solr.core.collection6 / com.codahale.metrics.MetricRegistry@68b63222
[junit4] 2> 2629622 INFO (TEST-TestLazyCores.testCachingLimit-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.SolrTestCaseJ4 ###Ending testCachingLimit
[junit4] 2> 2629624 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.SolrTestCaseJ4 ###Starting testMidUseUnload
[junit4] 2> 2629636 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/solr.xml
[junit4] 2> 2629639 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2629639 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2629639 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2629671 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629676 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629676 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629677 INFO (TEST-TestLazyCores.testMidUseUnload-seed#[CC480704D68B187D]) [ x:collection3] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2629692 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629692 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629693 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2629700 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=tiny
[junit4] 2> 2629700 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.s.IndexSchema [collection2] Schema name=tiny
[junit4] 2> 2629700 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.s.IndexSchema [collection5] Schema name=tiny
[junit4] 2> 2629702 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629702 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629702 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.s.IndexSchema Loaded schema tiny/1.1 with uniqueid field id
[junit4] 2> 2629702 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.c.CoreContainer Creating SolrCore 'collection5' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/./collection5, trusted=true
[junit4] 2> 2629702 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.c.CoreContainer Creating SolrCore 'collection2' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/./collection2, trusted=true
[junit4] 2> 2629702 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/./collection1, trusted=true
[junit4] 2> 2629703 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection5' (registry 'solr.core.collection5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629703 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection2' (registry 'solr.core.collection2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629703 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629703 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629703 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@36faf6ba
[junit4] 2> 2629703 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.c.SolrCore [[collection2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/collection2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/./collection2/data/]
[junit4] 2> 2629703 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.c.SolrCore [[collection5] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/collection5], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/./collection5/data/]
[junit4] 2> 2629703 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2629703 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/./collection1/data/]
[junit4] 2> 2629705 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2629705 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2629705 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1498734451, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2629758 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629758 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629758 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629758 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2629758 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629758 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2629758 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2629758 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2629758 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 2629759 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.s.SolrIndexSearcher Opening [Searcher@4583ecf1[collection5] main]
[junit4] 2> 2629759 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.s.SolrIndexSearcher Opening [Searcher@43610860[collection2] main]
[junit4] 2> 2629759 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@34320539[collection1] main]
[junit4] 2> 2629759 INFO (coreLoadExecutor-8948-thread-3-processing-x:collection3) [ x:collection5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/collection5/conf
[junit4] 2> 2629759 INFO (coreLoadExecutor-8948-thread-2-processing-x:collection3) [ x:collection2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_CC480704D68B187D-001/tempDir-003/collection2/conf
[junit4] 2> 2629759 INFO (coreLoadExecutor-8948-thread-1-processing-x:collection3) [ x:collection1] o.a.s.r.
[...truncated too long message...]
estLazyCores]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] 2> at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Oct 11, 2017 4:25:36 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=23808, name=searcherExecutor-9002-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] 2> Oct 11, 2017 4:25:39 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=23808, name=searcherExecutor-9002-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] 2> at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false): {}, locale=pt, timezone=SystemV/AST4
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=2,free=147332096,total=530579456
[junit4] 2> NOTE: All tests run in this JVM: [TestFieldTypeCollectionResource, TestCloudSchemaless, JsonLoaderTest, TestQueryUtils, DistributedDebugComponentTest, CreateCollectionCleanupTest, TestPushWriter, DistributedTermsComponentTest, MBeansHandlerTest, TestAnalyzedSuggestions, DistribJoinFromCollectionTest, TestRandomFaceting, CdcrUpdateLogTest, OpenCloseCoreStressTest, HdfsRestartWhileUpdatingTest, TestLuceneIndexBackCompat, QueryEqualityTest, TestSimpleTextCodec, FullSolrCloudDistribCmdsTest, LeaderElectionContextKeyTest, TestGraphMLResponseWriter, RecoveryAfterSoftCommitTest, ImplicitSnitchTest, SpellCheckCollatorWithCollapseTest, HdfsBasicDistributedZk2Test, TestLRUCache, TestDocBasedVersionConstraints, ManagedSchemaRoundRobinCloudTest, TestRangeQuery, TestAddFieldRealTimeGet, MultiThreadedOCPTest, TestFastWriter, DistributedFacetPivotLongTailTest, TestSimpleTrackingShardHandler, SolrCloudExampleTest, TestCharFilters, ShowFileRequestHandlerTest, LoggingHandlerTest, TestManagedSynonymGraphFilterFactory, PeerSyncWithIndexFingerprintCachingTest, OpenExchangeRatesOrgProviderTest, TestFiltering, CollectionsAPIDistributedZkTest, TestLegacyNumericUtils, TestSQLHandlerNonCloud, ConfigureRecoveryStrategyTest, TestBulkSchemaConcurrent, ShardRoutingCustomTest, SolrCmdDistributorTest, TestImpersonationWithHadoopAuth, TestSolrCloudWithKerberosAlt, SolrMetricManagerTest, TermVectorComponentDistributedTest, TestRestoreCore, PreAnalyzedFieldTest, WrapperMergePolicyFactoryTest, TestExactStatsCache, TestFilteredDocIdSet, SubstringBytesRefFilterTest, TestDistribDocBasedVersion, SolrPluginUtilsTest, TestFreeTextSuggestions, TestIntervalFaceting, TestFastOutputStream, BasicAuthIntegrationTest, CloudMLTQParserTest, TestQueryWrapperFilter, TestCustomStream, PropertiesRequestHandlerTest, HLLSerializationTest, TestPullReplica, TestManagedSchemaAPI, DistributedVersionInfoTest, HttpTriggerListenerTest, UUIDUpdateProcessorFallbackTest, DistributedIntervalFacetingTest, TestStressCloudBlindAtomicUpdates, TestCollationField, DeleteInactiveReplicaTest, TestJoin, TestPostingsSolrHighlighter, RulesTest, SystemInfoHandlerTest, SolrRequestParserTest, TestDistributedGrouping, TestEmbeddedSolrServerConstructors, HdfsLockFactoryTest, DistributedFacetPivotSmallTest, TestZkChroot, TestConfigSetImmutable, InfixSuggestersTest, TestMissingGroups, SpellCheckCollatorTest, BasicFunctionalityTest, ChaosMonkeySafeLeaderTest, TestLockTree, TestRTimerTree, TestSubQueryTransformer, BigEndianAscendingWordSerializerTest, AutoScalingHandlerTest, TestCustomSort, TestSubQueryTransformerDistrib, AsyncCallRequestStatusResponseTest, SparseHLLTest, UpdateParamsTest, DebugComponentTest, TestSolrDeletionPolicy1, TestFieldCacheVsDocValues, TestAuthenticationFramework, TestSchemalessBufferedUpdates, TestClassicSimilarityFactory, TestExclusionRuleCollectionAccess, TestBackupRepositoryFactory, TestSearcherReuse, CacheHeaderTest, TestSchemaSimilarityResource, MissingSegmentRecoveryTest, LeaderInitiatedRecoveryOnShardRestartTest, TestNamedUpdateProcessors, ConfigSetsAPITest, TestInPlaceUpdatesStandalone, AtomicUpdateProcessorFactoryTest, AutoscalingHistoryHandlerTest, ResponseHeaderTest, FieldMutatingUpdateProcessorTest, SolrCoreCheckLockOnStartupTest, CollectionsAPISolrJTest, TestPartialUpdateDeduplication, ZkStateReaderTest, TestInitParams, OverseerCollectionConfigSetProcessorTest, TestLuceneMatchVersion, TestCSVLoader, TestHdfsUpdateLog, CollectionReloadTest, DistributedQueryComponentCustomSortTest, TestStressLiveNodes, ShardSplitTest, TestTrackingShardHandlerFactory, ExternalFileFieldSortTest, CurrencyFieldTypeTest, TestCloudInspectUtil, TestIndexSearcher, HdfsCollectionsAPIDistributedZkTest, TestConfigReload, TestXmlQParserPlugin, SuggesterTSTTest, TestDistributedMissingSort, TestSha256AuthenticationProvider, TestTestInjection, TestReload, PKIAuthenticationIntegrationTest, TestUseDocValuesAsStored2, TestElisionMultitermQuery, ZkNodePropsTest, TermsComponentTest, SolrGraphiteReporterTest, MoveReplicaHDFSTest, TestZkAclsWithHadoopAuth, TestBadConfig, BasicZkTest, TestCoreDiscovery, TestDistributedStatsComponentCardinality, TestDocSet, TestSystemIdResolver, DocumentAnalysisRequestHandlerTest, SolrIndexConfigTest, OutOfBoxZkACLAndCredentialsProvidersTest, SolrJmxReporterTest, DeleteShardTest, TestScoreJoinQPScore, TestSSLRandomization, TestOrdValues, AnalyticsQueryTest, TestReplicationHandlerBackup, DocumentBuilderTest, TestSolrQueryResponse, TestClusterProperties, TestCloudRecovery, ClassificationUpdateProcessorFactoryTest, ClassificationUpdateProcessorTest, BasicDistributedZkTest, TestDistributedSearch, DistributedSpellCheckComponentTest, TestRealTimeGet, TestStressVersions, HardAutoCommitTest, TestSolr4Spatial, SolrCoreTest, StatsComponentTest, SpellCheckComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFunctionQuery, TestLazyCores]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestLazyCores -Dtests.seed=CC480704D68B187D -Dtests.slow=true -Dtests.locale=pt -Dtests.timezone=SystemV/AST4 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J1 | TestLazyCores (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:
[junit4] > 1) Thread[id=23808, name=searcherExecutor-9002-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([CC480704D68B187D]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=23808, name=searcherExecutor-9002-thread-1, state=WAITING, group=TGRP-TestLazyCores]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([CC480704D68B187D]:0)
[junit4] Completed [412/744 (1!)] on J1 in 27.68s, 9 tests, 3 errors <<< FAILURES!
[...truncated 47975 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 237 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/237/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
2 tests failed.
FAILED: org.apache.solr.cloud.LeaderElectionContextKeyTest.test
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([A0D88E37DE2D16A1:288CB1ED70D17B59]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at org.apache.solr.cloud.LeaderElectionContextKeyTest.test(LeaderElectionContextKeyTest.java:88)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover
Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:35320 within 30000 ms
Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:35320 within 30000 ms
at __randomizedtesting.SeedInfo.seed([A0D88E37DE2D16A1:15976EA2ABEE4FEE]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
at org.apache.solr.cloud.OutOfBoxZkACLAndCredentialsProvidersTest.setUp(OutOfBoxZkACLAndCredentialsProvidersTest.java:72)
at sun.reflect.GeneratedMethodAccessor48.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:35320 within 30000 ms
at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
... 41 more
Build Log:
[...truncated 13258 lines...]
[junit4] Suite: org.apache.solr.cloud.LeaderElectionContextKeyTest
[junit4] 2> 2591469 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/init-core-data-001
[junit4] 2> 2591469 WARN (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=58 numCloses=58
[junit4] 2> 2591469 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2591471 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2591472 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001
[junit4] 2> 2591472 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2591475 INFO (Thread-5674) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2591475 INFO (Thread-5674) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2591476 ERROR (Thread-5674) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2591579 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:33957
[junit4] 2> 2591656 INFO (jetty-launcher-3612-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2591657 INFO (jetty-launcher-3612-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@264d0b73{/solr,null,AVAILABLE}
[junit4] 2> 2591658 INFO (jetty-launcher-3612-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@760f5e59{HTTP/1.1,[http/1.1]}{127.0.0.1:43591}
[junit4] 2> 2591658 INFO (jetty-launcher-3612-thread-1) [ ] o.e.j.s.Server Started @2594340ms
[junit4] 2> 2591658 INFO (jetty-launcher-3612-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43591}
[junit4] 2> 2591658 ERROR (jetty-launcher-3612-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2591659 INFO (jetty-launcher-3612-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 2591659 INFO (jetty-launcher-3612-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2591659 INFO (jetty-launcher-3612-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 2591670 INFO (jetty-launcher-3612-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-12T01:47:29.346Z
[junit4] 2> 2591687 INFO (jetty-launcher-3612-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2591711 INFO (jetty-launcher-3612-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33957/solr
[junit4] 2> 2591712 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15f0e43dc000001, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2592175 INFO (jetty-launcher-3612-thread-1) [n:127.0.0.1:43591_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2592181 INFO (jetty-launcher-3612-thread-1) [n:127.0.0.1:43591_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43591_solr
[junit4] 2> 2592183 INFO (jetty-launcher-3612-thread-1) [n:127.0.0.1:43591_solr ] o.a.s.c.Overseer Overseer (id=98813401442025475-127.0.0.1:43591_solr-n_0000000000) starting
[junit4] 2> 2592260 INFO (jetty-launcher-3612-thread-1) [n:127.0.0.1:43591_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43591_solr
[junit4] 2> 2592266 INFO (zkCallback-3619-thread-1-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2592394 INFO (jetty-launcher-3612-thread-1) [n:127.0.0.1:43591_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43591.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4915d672
[junit4] 2> 2592403 INFO (jetty-launcher-3612-thread-1) [n:127.0.0.1:43591_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43591.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4915d672
[junit4] 2> 2592404 INFO (jetty-launcher-3612-thread-1) [n:127.0.0.1:43591_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43591.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4915d672
[junit4] 2> 2592405 INFO (jetty-launcher-3612-thread-1) [n:127.0.0.1:43591_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001/node1/.
[junit4] 2> 2592482 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2592499 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33957/solr ready
[junit4] 2> 2592524 INFO (qtp478359447-22771) [n:127.0.0.1:43591_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2592530 INFO (OverseerThreadFactory-9531-thread-1-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.CreateCollectionCmd Create collection testCollection1
[junit4] 2> 2592648 INFO (OverseerStateUpdate-98813401442025475-127.0.0.1:43591_solr-n_0000000000) [n:127.0.0.1:43591_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testCollection1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testCollection1_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:43591/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 2592653 INFO (OverseerStateUpdate-98813401442025475-127.0.0.1:43591_solr-n_0000000000) [n:127.0.0.1:43591_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testCollection1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"testCollection1_shard2_replica_n3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:43591/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 2592857 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=config&newCollection=true&name=testCollection1_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2592858 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2592858 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection1_shard2_replica_n3&action=CREATE&numShards=2&collection=testCollection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2592966 INFO (zkCallback-3619-thread-1-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2592967 INFO (zkCallback-3619-thread-2-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2593880 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2593881 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2593887 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.IndexSchema [testCollection1_shard2_replica_n3] Schema name=minimal
[junit4] 2> 2593888 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.IndexSchema [testCollection1_shard1_replica_n1] Schema name=minimal
[junit4] 2> 2593889 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2593889 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'testCollection1_shard2_replica_n3' using configuration from collection testCollection1, trusted=true
[junit4] 2> 2593890 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43591.solr.core.testCollection1.shard2.replica_n3' (registry 'solr.core.testCollection1.shard2.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4915d672
[junit4] 2> 2593891 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2593891 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrCore [[testCollection1_shard2_replica_n3] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001/node1/testCollection1_shard2_replica_n3], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001/node1/./testCollection1_shard2_replica_n3/data/]
[junit4] 2> 2593892 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2593892 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testCollection1_shard1_replica_n1' using configuration from collection testCollection1, trusted=true
[junit4] 2> 2593894 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43591.solr.core.testCollection1.shard1.replica_n1' (registry 'solr.core.testCollection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4915d672
[junit4] 2> 2593894 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2593894 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore [[testCollection1_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001/node1/testCollection1_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001/node1/./testCollection1_shard1_replica_n1/data/]
[junit4] 2> 2593955 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2593955 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2593957 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2593957 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2593958 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@6dd5e884[testCollection1_shard2_replica_n3] main]
[junit4] 2> 2593960 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 2593960 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 2593961 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2593961 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581014425678118912
[junit4] 2> 2593962 INFO (searcherExecutor-9534-thread-1-processing-n:127.0.0.1:43591_solr x:testCollection1_shard2_replica_n3 s:shard2 c:testCollection1 r:core_node4) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrCore [testCollection1_shard2_replica_n3] Registered new searcher Searcher@6dd5e884[testCollection1_shard2_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2593966 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2593966 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2593967 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2593967 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2593967 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@782b9cde[testCollection1_shard1_replica_n1] main]
[junit4] 2> 2593967 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2593967 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2593967 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43591/solr/testCollection1_shard2_replica_n3/
[junit4] 2> 2593967 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2593967 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SyncStrategy http://127.0.0.1:43591/solr/testCollection1_shard2_replica_n3/ has no replicas
[junit4] 2> 2593967 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2593972 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 2593973 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 2593974 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2593974 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581014425691750400
[junit4] 2> 2593975 INFO (searcherExecutor-9535-thread-1-processing-n:127.0.0.1:43591_solr x:testCollection1_shard1_replica_n1 s:shard1 c:testCollection1 r:core_node2) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore [testCollection1_shard1_replica_n1] Registered new searcher Searcher@782b9cde[testCollection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2593977 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43591/solr/testCollection1_shard2_replica_n3/ shard2
[junit4] 2> 2593982 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2593982 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2593982 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43591/solr/testCollection1_shard1_replica_n1/
[junit4] 2> 2593982 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2593982 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:43591/solr/testCollection1_shard1_replica_n1/ has no replicas
[junit4] 2> 2593982 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2593995 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43591/solr/testCollection1_shard1_replica_n1/ shard1
[junit4] 2> 2594100 INFO (zkCallback-3619-thread-1-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2594101 INFO (zkCallback-3619-thread-2-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2594137 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2594140 INFO (qtp478359447-22763) [n:127.0.0.1:43591_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection1_shard2_replica_n3&action=CREATE&numShards=2&collection=testCollection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1282
[junit4] 2> 2594150 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2594167 INFO (qtp478359447-22761) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=config&newCollection=true&name=testCollection1_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1310
[junit4] 2> 2594186 INFO (qtp478359447-22771) [n:127.0.0.1:43591_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2594283 INFO (zkCallback-3619-thread-1-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2594283 INFO (zkCallback-3619-thread-2-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2594535 INFO (OverseerCollectionConfigSetProcessor-98813401442025475-127.0.0.1:43591_solr-n_0000000000) [n:127.0.0.1:43591_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2595187 INFO (qtp478359447-22771) [n:127.0.0.1:43591_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2663
[junit4] 2> 2595192 INFO (qtp478359447-22768) [n:127.0.0.1:43591_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection2&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2595199 INFO (OverseerThreadFactory-9531-thread-2-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.CreateCollectionCmd Create collection testCollection2
[junit4] 2> 2595412 INFO (OverseerStateUpdate-98813401442025475-127.0.0.1:43591_solr-n_0000000000) [n:127.0.0.1:43591_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testCollection2",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testCollection2_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:43591/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 2595419 INFO (OverseerStateUpdate-98813401442025475-127.0.0.1:43591_solr-n_0000000000) [n:127.0.0.1:43591_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testCollection2",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"testCollection2_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:43591/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 2595618 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=testCollection2_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection2&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2595618 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection2_shard2_replica_n2&action=CREATE&numShards=2&collection=testCollection2&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2595725 INFO (zkCallback-3619-thread-2-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2595725 INFO (zkCallback-3619-thread-1-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2596639 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2596639 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 2596645 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.IndexSchema [testCollection2_shard1_replica_n1] Schema name=minimal
[junit4] 2> 2596645 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.IndexSchema [testCollection2_shard2_replica_n2] Schema name=minimal
[junit4] 2> 2596646 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2596647 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testCollection2_shard1_replica_n1' using configuration from collection testCollection2, trusted=true
[junit4] 2> 2596646 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2596647 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'testCollection2_shard2_replica_n2' using configuration from collection testCollection2, trusted=true
[junit4] 2> 2596647 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43591.solr.core.testCollection2.shard1.replica_n1' (registry 'solr.core.testCollection2.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4915d672
[junit4] 2> 2596647 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43591.solr.core.testCollection2.shard2.replica_n2' (registry 'solr.core.testCollection2.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@4915d672
[junit4] 2> 2596647 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2596647 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2596647 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore [[testCollection2_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001/node1/testCollection2_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001/node1/./testCollection2_shard1_replica_n1/data/]
[junit4] 2> 2596647 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore [[testCollection2_shard2_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001/node1/testCollection2_shard2_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001/tempDir-001/node1/./testCollection2_shard2_replica_n2/data/]
[junit4] 2> 2596694 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2596694 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2596695 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2596695 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2596695 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2596695 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2596696 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2596696 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2596696 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@3ea5d6[testCollection2_shard2_replica_n2] main]
[junit4] 2> 2596697 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@625efe8[testCollection2_shard1_replica_n1] main]
[junit4] 2> 2596698 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 2596698 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
[junit4] 2> 2596698 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 2596698 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
[junit4] 2> 2596699 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2596699 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2596699 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581014428549120000
[junit4] 2> 2596699 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1581014428549120000
[junit4] 2> 2596699 INFO (searcherExecutor-9545-thread-1-processing-n:127.0.0.1:43591_solr x:testCollection2_shard2_replica_n2 s:shard2 c:testCollection2 r:core_node4) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore [testCollection2_shard2_replica_n2] Registered new searcher Searcher@3ea5d6[testCollection2_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2596700 INFO (searcherExecutor-9544-thread-1-processing-n:127.0.0.1:43591_solr x:testCollection2_shard1_replica_n1 s:shard1 c:testCollection2 r:core_node3) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore [testCollection2_shard1_replica_n1] Registered new searcher Searcher@625efe8[testCollection2_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2596709 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2596709 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2596709 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43591/solr/testCollection2_shard1_replica_n1/
[junit4] 2> 2596709 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2596709 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:43591/solr/testCollection2_shard1_replica_n1/ has no replicas
[junit4] 2> 2596709 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2596712 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2596712 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2596713 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43591/solr/testCollection2_shard2_replica_n2/
[junit4] 2> 2596713 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2596713 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:43591/solr/testCollection2_shard2_replica_n2/ has no replicas
[junit4] 2> 2596713 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2596718 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43591/solr/testCollection2_shard1_replica_n1/ shard1
[junit4] 2> 2596721 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43591/solr/testCollection2_shard2_replica_n2/ shard2
[junit4] 2> 2596825 INFO (zkCallback-3619-thread-2-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2596825 INFO (zkCallback-3619-thread-1-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2596872 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2596874 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2596875 INFO (qtp478359447-22767) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=testCollection2_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection2&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1257
[junit4] 2> 2596876 INFO (qtp478359447-22764) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection2_shard2_replica_n2&action=CREATE&numShards=2&collection=testCollection2&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1258
[junit4] 2> 2596880 INFO (qtp478359447-22768) [n:127.0.0.1:43591_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2596983 INFO (zkCallback-3619-thread-1-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2596983 INFO (zkCallback-3619-thread-2-processing-n:127.0.0.1:43591_solr) [n:127.0.0.1:43591_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2597204 INFO (OverseerCollectionConfigSetProcessor-98813401442025475-127.0.0.1:43591_solr-n_0000000000) [n:127.0.0.1:43591_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2597883 INFO (qtp478359447-22768) [n:127.0.0.1:43591_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection2&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2692
[junit4] 2> 2597883 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: testCollection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 2597884 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: testCollection1
[junit4] 2> 2597884 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: testCollection2 failOnTimeout:true timeout (sec):30
[junit4] 2> 2597884 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: testCollection2
[junit4] 2> 2597898 INFO (TEST-LeaderElectionContextKeyTest.test-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2597904 INFO (qtp478359447-22772) [n:127.0.0.1:43591_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection1.shard2.replica_n3, tag=null
[junit4] 2> 2597904 INFO (qtp478359447-22772) [n:127.0.0.1:43591_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@501a2073: rootName = solr_43591, domain = solr.core.testCollection1.shard2.replica_n3, service url = null, agent id = null] for registry solr.core.testCollection1.shard2.replica_n3 / com.codahale.metrics.MetricRegistry@182c2f15
[junit4] 2> 2597910 INFO (qtp478359447-22772) [n:127.0.0.1:43591_solr ] o.a.s.c.SolrCore [testCollection1_shard2_replica_n3] CLOSING SolrCore org.apache.solr.core.SolrCore@10394412
[junit4] 2> 2597910 INFO (qtp478359447-22772) [n:127.0.0.1:43591_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection1.shard2.replica_n3, tag=272188434
[junit4] 2> 2597910 INFO (qtp478359447-22772) [n:127.0.0.1:43591_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection1.shard2.leader, tag=272188434
[junit4] 2> 2597913 INFO (qtp478359447-22772) [n:127.0.0.1:43591_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=false&core=testCollection1_shard2_replica_n3&deleteDataDir=false&action=UNLOAD&indexInfo=true&wt=javabin&version=2&deleteIndex=false} status=0 QTime=10
[junit4] 2> 2597914 INFO (TEST-LeaderElectionContextKeyTest.test-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.SolrTestCaseJ4 ###Ending test
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=LeaderElectionContextKeyTest -Dtests.method=test -Dtests.seed=A0D88E37DE2D16A1 -Dtests.slow=true -Dtests.locale=sv -Dtests.timezone=Europe/Malta -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 0.03s J1 | LeaderElectionContextKeyTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError
[junit4] > at __randomizedtesting.SeedInfo.seed([A0D88E37DE2D16A1:288CB1ED70D17B59]:0)
[junit4] > at org.apache.solr.cloud.LeaderElectionContextKeyTest.test(LeaderElectionContextKeyTest.java:88)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2597920 INFO (jetty-closer-3613-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@760f5e59{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 2597920 INFO (jetty-closer-3613-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=957425942
[junit4] 2> 2597921 INFO (jetty-closer-3613-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 2597921 INFO (jetty-closer-3613-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4741240b: rootName = solr_43591, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@4c1f4d0d
[junit4] 2> 2597924 INFO (jetty-closer-3613-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 2597924 INFO (jetty-closer-3613-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@65961d4f: rootName = solr_43591, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@6621f48
[junit4] 2> 2597926 INFO (jetty-closer-3613-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 2597926 INFO (jetty-closer-3613-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1ea1e4b8: rootName = solr_43591, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@14c924d
[junit4] 2> 2597928 INFO (jetty-closer-3613-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 2597928 INFO (coreCloseExecutor-9554-thread-1) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore [testCollection1_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@4ba47d02
[junit4] 2> 2597929 INFO (coreCloseExecutor-9554-thread-2) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore [testCollection2_shard2_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@1ca0dbc9
[junit4] 2> 2597929 INFO (coreCloseExecutor-9554-thread-1) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection1.shard1.replica_n1, tag=1269071106
[junit4] 2> 2597929 INFO (coreCloseExecutor-9554-thread-1) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3699990: rootName = solr_43591, domain = solr.core.testCollection1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testCollection1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@137c6c77
[junit4] 2> 2597928 INFO (coreCloseExecutor-9554-thread-3) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore [testCollection2_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@1ece7cd9
[junit4] 2> 2597934 INFO (coreCloseExecutor-9554-thread-1) [n:127.0.0.1:43591_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection1.shard1.leader, tag=1269071106
[junit4] 2> 2597934 INFO (coreCloseExecutor-9554-thread-2) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection2.shard2.replica_n2, tag=480304073
[junit4] 2> 2597934 INFO (coreCloseExecutor-9554-thread-2) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1b50f841: rootName = solr_43591, domain = solr.core.testCollection2.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.testCollection2.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@73299e08
[junit4] 2> 2597950 INFO (coreCloseExecutor-9554-thread-2) [n:127.0.0.1:43591_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection2.shard2.leader, tag=480304073
[junit4] 2> 2597950 INFO (coreCloseExecutor-9554-thread-3) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection2.shard1.replica_n1, tag=516848857
[junit4] 2> 2597950 INFO (coreCloseExecutor-9554-thread-3) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@70db3ef5: rootName = solr_43591, domain = solr.core.testCollection2.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testCollection2.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@6a809f9
[junit4] 2> 2597957 INFO (coreCloseExecutor-9554-thread-3) [n:127.0.0.1:43591_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection2.shard1.leader, tag=516848857
[junit4] 2> 2597959 INFO (jetty-closer-3613-thread-1) [ ] o.a.s.c.Overseer Overseer (id=98813401442025475-127.0.0.1:43591_solr-n_0000000000) closing
[junit4] 2> 2597959 INFO (OverseerStateUpdate-98813401442025475-127.0.0.1:43591_solr-n_0000000000) [n:127.0.0.1:43591_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43591_solr
[junit4] 2> 2597959 WARN (OverseerAutoScalingTriggerThread-98813401442025475-127.0.0.1:43591_solr-n_0000000000) [n:127.0.0.1:43591_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 2597963 INFO (jetty-closer-3613-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@264d0b73{/solr,null,UNAVAILABLE}
[junit4] 2> 2597964 ERROR (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2597964 INFO (SUITE-LeaderElectionContextKeyTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:33957 33957
[junit4] 2> 2600824 INFO (Thread-5674) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:33957 33957
[junit4] 2> 2600831 WARN (Thread-5674) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/aliases.json
[junit4] 2> 2 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/collections/testCollection2/state.json
[junit4] 2> 5 /solr/collections/testCollection1/state.json
[junit4] 2> 2 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/live_nodes
[junit4] 2> 2 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_A0D88E37DE2D16A1-001
[junit4] 2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=9, maxDocsPerChunk=8, blockSize=145), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=9, blockSize=145)), sim=RandomSimilarity(queryNorm=false): {}, locale=sv, timezone=Europe/Malta
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=244006496,total=517996544
[junit4] 2> NOTE: All tests run in this JVM: [OverseerTaskQueueTest, SolrGraphiteReporterTest, TestSimpleTrackingShardHandler, TestQueryUtils, TestDistribIDF, DirectSolrSpellCheckerTest, HdfsDirectoryFactoryTest, HdfsCollectionsAPIDistributedZkTest, JSONWriterTest, PeerSyncWithIndexFingerprintCachingTest, CoreAdminRequestStatusTest, DeleteInactiveReplicaTest, TestWordDelimiterFilterFactory, AutoAddReplicasIntegrationTest, SpellCheckCollatorWithCollapseTest, TestLuceneMatchVersion, TestRTGBase, SpellPossibilityIteratorTest, ConfigureRecoveryStrategyTest, BadIndexSchemaTest, TestOrdValues, AtomicUpdateProcessorFactoryTest, TestPullReplica, SignatureUpdateProcessorFactoryTest, SpellingQueryConverterTest, TestFastLRUCache, BlockCacheTest, TestRandomFaceting, RecoveryAfterSoftCommitTest, TestIntervalFaceting, TestPointFields, ReplaceNodeTest, SpellCheckComponentTest, DebugComponentTest, IndexSchemaRuntimeFieldTest, DocumentAnalysisRequestHandlerTest, TestReloadAndDeleteDocs, TestSolrConfigHandlerConcurrent, TestJettySolrRunner, TestSchemaSimilarityResource, TestDocBasedVersionConstraints, ForceLeaderTest, CollectionReloadTest, ConvertedLegacyTest, TestLeaderElectionWithEmptyReplica, LeaderFailureAfterFreshStartTest, TestPushWriter, TestSolrXml, SystemInfoHandlerTest, TestPayloadCheckQParserPlugin, SubstringBytesRefFilterTest, TestDistributedGrouping, TermVectorComponentDistributedTest, TestZkChroot, BasicZkTest, CurrencyFieldTypeTest, UpdateRequestProcessorFactoryTest, LukeRequestHandlerTest, CopyFieldTest, TestManagedStopFilterFactory, TestSQLHandlerNonCloud, PolyFieldTest, ResponseHeaderTest, TestTolerantUpdateProcessorCloud, BufferStoreTest, TestCloudInspectUtil, TestUseDocValuesAsStored2, ZkSolrClientTest, DistributedVersionInfoTest, ExplicitHLLTest, TestScoreJoinQPScore, HttpTriggerListenerTest, TestManagedResourceStorage, TestImplicitCoreProperties, DistributedSuggestComponentTest, TestPartialUpdateDeduplication, TestSlowCompositeReaderWrapper, DistributedMLTComponentTest, ClassificationUpdateProcessorTest, SolrPluginUtilsTest, StatelessScriptUpdateProcessorFactoryTest, TestBadConfig, TestCharFilters, MultiThreadedOCPTest, CollectionTooManyReplicasTest, StatsComponentTest, SuggestComponentContextFilterQueryTest, PKIAuthenticationIntegrationTest, TestApiFramework, TestCloudPivotFacet, TestFastOutputStream, TestInPlaceUpdatesStandalone, TestSSLRandomization, SolrCloudReportersTest, DocumentBuilderTest, TestMacroExpander, QueryEqualityTest, StressHdfsTest, AsyncCallRequestStatusResponseTest, TestFieldCacheVsDocValues, TestFiltering, SynonymTokenizerTest, FileUtilsTest, HdfsRestartWhileUpdatingTest, TestCloudSchemaless, TestSchemalessBufferedUpdates, MissingSegmentRecoveryTest, TestMissingGroups, TestDistribDocBasedVersion, TestConfigSetImmutable, TlogReplayBufferedWhileIndexingTest, TestLuceneIndexBackCompat, MoveReplicaHDFSTest, ConfigSetsAPITest, PreAnalyzedFieldTest, RulesTest, TestDynamicFieldCollectionResource, CdcrVersionReplicationTest, ParsingFieldUpdateProcessorsTest, RankQueryTest, TestConfigReload, SolrRequestParserTest, TestAnalyzedSuggestions, AssignBackwardCompatibilityTest, TestNumericTerms64, SolrJmxReporterTest, RestartWhileUpdatingTest, TestCustomStream, TestSolrIndexConfig, CdcrReplicationHandlerTest, TestRawTransformer, TestDistributedMissingSort, CollectionsAPIDistributedZkTest, ShufflingReplicaListTransformerTest, TestTestInjection, HLLSerializationTest, ClassificationUpdateProcessorFactoryTest, OpenCloseCoreStressTest, SegmentsInfoRequestHandlerTest, ZkNodePropsTest, DOMUtilTest, AutoscalingHistoryHandlerTest, TestLMJelinekMercerSimilarityFactory, TestHdfsUpdateLog, TestSearcherReuse, TestFastWriter, DirectoryFactoryTest, BlockJoinFacetRandomTest, OpenExchangeRatesOrgProviderTest, TestFieldCacheSort, CdcrUpdateLogTest, TestBinaryResponseWriter, TestTrieFacet, ZkStateReaderTest, TestJsonFacets, DistributedQueryElevationComponentTest, TestLRUCache, DistributedFacetPivotLongTailTest, HdfsBasicDistributedZk2Test, DocExpirationUpdateProcessorFactoryTest, CdcrBootstrapTest, SpellCheckCollatorTest, HighlighterMaxOffsetTest, TestSimpleTextCodec, TestRequestForwarding, TestCryptoKeys, ManagedSchemaRoundRobinCloudTest, TestSchemaNameResource, FieldMutatingUpdateProcessorTest, DistribJoinFromCollectionTest, InfixSuggestersTest, AddBlockUpdateTest, TermVectorComponentTest, BadComponentTest, TestComplexPhraseLeadingWildcard, MBeansHandlerTest, TestCollationField, TestOnReconnectListenerSupport, TestFieldCollectionResource, TestHashQParserPlugin, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, FullSolrCloudDistribCmdsTest, LeaderElectionTest, ZkCLITest, ShardRoutingCustomTest, TestDistributedSearch, TestFaceting, TestHashPartitioner, TestRealTimeGet, TestJoin, TestReload, TestStressVersions, DistributedTermsComponentTest, TestRangeQuery, SolrCoreTest, TestGroupingSearch, PeerSyncTest, ShowFileRequestHandlerTest, AnalysisAfterCoreReloadTest, SuggesterFSTTest, TestFoldingMultitermQuery, SuggesterTest, TestTrie, TestCSVLoader, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, DirectUpdateHandlerOptimizeTest, SortByFunctionTest, DocValuesMultiTest, DistanceFunctionTest, SolrInfoBeanTest, TestSurroundQueryParser, TestQueryTypes, TermsComponentTest, RequiredFieldsTest, TestSolrQueryParser, FastVectorHighlighterTest, TestJmxIntegration, TestCSVResponseWriter, UniqFieldsUpdateProcessorFactoryTest, TestPHPSerializedResponseWriter, PingRequestHandlerTest, SOLR749Test, UpdateParamsTest, TestStressRecovery, EchoParamsTest, TestDFRSimilarityFactory, TestLMDirichletSimilarityFactory, TestBM25SimilarityFactory, TimeZoneUtilsTest, TestSystemIdResolver, ClusterStateTest, TestUtils, SliceStateTest, TestCrossCoreJoin, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, ActionThrottleTest, AliasIntegrationTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, ConcurrentDeleteAndCreateCollectionTest, CreateCollectionCleanupTest, CustomCollectionTest, DeleteNodeTest, DeleteShardTest, DeleteStatusTest, DistribCursorPagingTest, LeaderElectionContextKeyTest]
[junit4] Completed [520/744 (1!)] on J1 in 9.40s, 1 test, 1 failure <<< FAILURES!
[...truncated 12 lines...]
[junit4] Suite: org.apache.solr.cloud.OutOfBoxZkACLAndCredentialsProvidersTest
[junit4] 2> 2649969 INFO (SUITE-OutOfBoxZkACLAndCredentialsProvidersTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.OutOfBoxZkACLAndCredentialsProvidersTest_A0D88E37DE2D16A1-001/init-core-data-001
[junit4] 2> 2649970 WARN (SUITE-OutOfBoxZkACLAndCredentialsProvidersTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=16 numCloses=16
[junit4] 2> 2649970 INFO (SUITE-OutOfBoxZkACLAndCredentialsProvidersTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2649971 INFO (SUITE-OutOfBoxZkACLAndCredentialsProvidersTest-seed#[A0D88E37DE2D16A1]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2649978 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOutOfBoxSolrZkClient-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testOutOfBoxSolrZkClient
[junit4] 2> 2649978 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOutOfBoxSolrZkClient-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.OutOfBoxZkACLAndCredentialsProvidersTest ####SETUP_START testOutOfBoxSolrZkClient
[junit4] 2> 2649978 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOutOfBoxSolrZkClient-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.OutOfBoxZkACLAndCredentialsProvidersTest ZooKeeper dataDir:/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.OutOfBoxZkACLAndCredentialsProvidersTest_A0D88E37DE2D16A1-001/tempDir-002/zookeeper/server1/data
[junit4] 2> 2649978 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOutOfBoxSolrZkClient-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2649979 INFO (Thread-6198) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2649979 INFO (Thread-6198) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2649982 ERROR (Thread-6198) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2650079 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOutOfBoxSolrZkClient-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.ZkTestServer start zk server on port:47206
[junit4] 2> 2650098 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOutOfBoxSolrZkClient-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.OutOfBoxZkACLAndCredentialsProvidersTest ####SETUP_END testOutOfBoxSolrZkClient
[junit4] 2> 2650123 ERROR (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOutOfBoxSolrZkClient-seed#[A0D88E37DE2D16A1]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2650127 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOutOfBoxSolrZkClient-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:47206 47206
[junit4] 2> 2658790 INFO (Thread-6198) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:47206 47206
[junit4] 2> 2658803 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOutOfBoxSolrZkClient-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testOutOfBoxSolrZkClient
[junit4] 2> 2658813 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testOpenACLUnsafeAllover
[junit4] 2> 2658813 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.OutOfBoxZkACLAndCredentialsProvidersTest ####SETUP_START testOpenACLUnsafeAllover
[junit4] 2> 2658814 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.OutOfBoxZkACLAndCredentialsProvidersTest ZooKeeper dataDir:/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.OutOfBoxZkACLAndCredentialsProvidersTest_A0D88E37DE2D16A1-001/tempDir-004/zookeeper/server1/data
[junit4] 2> 2658814 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2658815 INFO (Thread-6199) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2658815 INFO (Thread-6199) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2658820 ERROR (Thread-6199) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2658915 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.ZkTestServer start zk server on port:35320
[junit4] 2> 2690064 ERROR (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover-seed#[A0D88E37DE2D16A1]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2690069 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35320 35320
[junit4] 2> 2696250 INFO (Thread-6199) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35320 35320
[junit4] 2> 2696258 INFO (TEST-OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover-seed#[A0D88E37DE2D16A1]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testOpenACLUnsafeAllover
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=OutOfBoxZkACLAndCredentialsProvidersTest -Dtests.method=testOpenACLUnsafeAllover -Dtests.seed=A0D88E37DE2D16A1 -Dtests.slow=true -Dtests.locale=es -Dtests.timezone=Africa/Kigali -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 37.5s J0 | OutOfBoxZkACLAndCredentialsProvidersTest.testOpenACLUnsafeAllover <<<
[junit4] > Throwable #1: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:35320 within 30000 ms
[junit4] > at __randomizedtesting.SeedInfo.seed([A0D88E37DE2D16A1:15976EA2ABEE4FEE]:0)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:183)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
[junit4] > at org.apache.solr.cloud.OutOfBoxZkACLAndCredentialsProvidersTest.setUp(OutOfBoxZkACLAndCredentialsProvidersTest.java:72)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:35320 within 30000 ms
[junit4] > at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:175)
[junit4] > ... 41 more
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.OutOfBoxZkACLAndCredentialsProvidersTest_A0D88E37DE2D16A1-001
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=es, timezone=Africa/Kigali
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=161294168,total=522190848
[junit4] 2> NOTE: All tests run in this JVM: [PeerSyncReplicationTest, SolrJmxReporterCloudTest, TestRandomDVFaceting, TestPayloadScoreQParserPlugin, TestLRUStatsCache, WordBreakSolrSpellCheckerTest, AlternateDirectoryTest, SystemLogListenerTest, TestWriterPerf, SolrIndexSplitterTest, TestReqParamsAPI, DeleteLastCustomShardedReplicaTest, TestReversedWildcardFilterFactory, UnloadDistributedZkTest, TestAnalyzeInfixSuggestions, PathHierarchyTokenizerFactoryTest, TestRandomCollapseQParserPlugin, TestFileDictionaryLookup, ZkStateWriterTest, TestSchemaVersionResource, BlockJoinFacetSimpleTest, AtomicUpdatesTest, TestLeaderElectionZkExpiry, RegexBoostProcessorTest, TestFuzzyAnalyzedSuggestions, TestCollapseQParserPlugin, TestFieldCacheSortRandom, TestHighlightDedupGrouping, RecoveryZkTest, TestManagedSchema, RemoteQueryErrorTest, SpatialHeatmapFacetsTest, StatsReloadRaceTest, DocValuesTest, TestQuerySenderNoQuery, TestDynamicLoading, TestBlobHandler, ConnectionReuseTest, TestDynamicFieldResource, TestInPlaceUpdatesDistrib, HttpPartitionTest, TestExactSharedStatsCache, ClusterStateUpdateTest, CursorPagingTest, TestDeleteCollectionOnDownNodes, MigrateRouteKeyTest, TestCustomDocTransformer, TestNRTOpen, CoreAdminHandlerTest, TestMultiWordSynonyms, SimpleFacetsTest, SolrTestCaseJ4Test, TestPivotHelperCode, TestSolrCloudWithSecureImpersonation, CdcrReplicationDistributedZkTest, BadCopyFieldTest, CircularListTest, TestSystemCollAutoCreate, TestManagedSynonymFilterFactory, TestFieldTypeResource, RequestLoggingTest, MultiTermTest, ConjunctionSolrSpellCheckerTest, TestSolrCloudWithDelegationTokens, TestFieldCacheWithThreads, SolrXmlInZkTest, TestOmitPositions, TriLevelCompositeIdRoutingTest, DocValuesNotIndexedTest, FullHLLTest, TestDFISimilarityFactory, ZkFailoverTest, TestRetrieveFieldsOptimizer, TestConfigSetProperties, DistributedSpellCheckComponentTest, URLClassifyProcessorTest, DistributedFacetPivotSmallAdvancedTest, CloneFieldUpdateProcessorFactoryTest, PrimUtilsTest, TemplateUpdateProcessorTest, RAMDirectoryFactoryTest, TestTolerantSearch, OverriddenZkACLAndCredentialsProvidersTest, CollectionStateFormat2Test, SuggestComponentTest, TestExpandComponent, TestRuleBasedAuthorizationPlugin, CoreMergeIndexesAdminHandlerTest, SyncSliceTest, SimplePostToolTest, TestUpdate, TestPullReplicaErrorHandling, SolrShardReporterTest, SolrIndexMetricsTest, TestStreamBody, UUIDFieldTest, HdfsRecoveryZkTest, BaseCdcrDistributedZkTest, DataDrivenBlockJoinTest, TestExtendedDismaxParser, PrimitiveFieldTypeTest, DistanceUnitsTest, HdfsBasicDistributedZkTest, SpatialRPTFieldTypeTest, TestManagedSchemaThreadSafety, MoveReplicaTest, TestLFUCache, TestConfigSetsAPI, ResourceLoaderTest, TestStressInPlaceUpdates, TestClusterStateMutator, TestEmbeddedSolrServerAdminHandler, NodeMutatorTest, ConnectionManagerTest, NotRequiredUniqueKeyTest, HdfsWriteToMultipleCollectionsTest, TestSubQueryTransformerCrossCore, ChaosMonkeyNothingIsSafeWithPullReplicasTest, PreAnalyzedUpdateProcessorTest, AnalyticsMergeStrategyTest, BinaryUpdateRequestHandlerTest, IndexBasedSpellCheckerTest, HdfsDirectoryTest, AssignTest, DirectUpdateHandlerTest, SolrSlf4jReporterTest, ReplicationFactorTest, TestConfigOverlay, TestLazyCores, ChaosMonkeyNothingIsSafeTest, TestGeoJSONResponseWriter, TestCursorMarkWithoutUniqueKey, CollectionsAPIAsyncDistributedZkTest, ResponseLogComponentTest, UtilsToolTest, HLLUtilTest, ClassificationUpdateProcessorIntegrationTest, BlobRepositoryCloudTest, XsltUpdateRequestHandlerTest, TestTolerantUpdateProcessorRandomCloud, DateMathParserTest, TestReplicationHandler, TestPerFieldSimilarityWithDefaultOverride, TestIndexingPerformance, ScriptEngineTest, TestStressReorder, TestObjectReleaseTracker, RuleEngineTest, TestScoreJoinQPNoScore, IndexSchemaTest, AnalysisErrorHandlingTest, SoftAutoCommitTest, TestMacros, JvmMetricsTest, TestMinMaxOnMultiValuedField, HdfsSyncSliceTest, BlockJoinFacetDistribTest, DistributedQueryComponentOptimizationTest, TestGraphTermsQParserPlugin, DistributedExpandComponentTest, HdfsAutoAddReplicasIntegrationTest, DefaultValueUpdateProcessorTest, CdcrRequestHandlerTest, SuggesterWFSTTest, UninvertDocValuesMergePolicyTest, TestInfoStreamLogging, TestPrepRecovery, TestCollectionAPI, EnumFieldTest, TestCopyFieldCollectionResource, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DistributedQueueTest, QueryElevationComponentTest, HardAutoCommitTest, JavabinLoaderTest, SecurityConfHandlerTest, MergeStrategyTest, V2StandaloneTest, TestBinaryField, TestHdfsCloudBackupRestore, TestManagedResource, TestFieldSortValues, AutoCommitTest, TestRecovery, ZkControllerTest, TestCoreContainer, TestSolr4Spatial, SolrCmdDistributorTest, TestSort, TestFunctionQuery, BasicFunctionalityTest, TestIndexSearcher, HighlighterTest, TestCoreDiscovery, SuggesterTSTTest, TestStressLucene, SpatialFilterTest, SchemaVersionSpecificBehaviorTest, TestAtomicUpdateErrorCases, TestRemoteStreaming, TestSolrDeletionPolicy1, CacheHeaderTest, DisMaxRequestHandlerTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, TestValueSourceCache, MoreLikeThisHandlerTest, FieldAnalysisRequestHandlerTest, LoggingHandlerTest, ReturnFieldsTest, QueryParsingTest, JsonLoaderTest, CSVRequestHandlerTest, TestComponentsName, SearchHandlerTest, HighlighterConfigTest, TestQuerySenderListener, SolrIndexConfigTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestDocSet, TestSearchPerf, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, ExternalFileFieldSortTest, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestPhraseSuggestions, TestCodecSupport, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestIBSimilarityFactory, PluginInfoTest, TestSuggestSpellingConverter, DateFieldTest, TestSolrJ, TestDocumentBuilder, DistributedIntervalFacetingTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CollectionsAPISolrJTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, LeaderFailoverAfterPartitionTest, OutOfBoxZkACLAndCredentialsProvidersTest]
[junit4] Completed [524/744 (2!)] on J0 in 46.31s, 2 tests, 1 error <<< FAILURES!
[...truncated 47646 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 236 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/236/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:132) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:108) at sun.reflect.GeneratedConstructorAccessor611.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:774) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:836) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1089) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:844) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1029) at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:642) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:132)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:108)
at sun.reflect.GeneratedConstructorAccessor611.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:774)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:836)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1089)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:844)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1029)
at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:642)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([F2218DA170BE7B3C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:294)
at sun.reflect.GeneratedMethodAccessor162.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13422 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/init-core-data-001
[junit4] 2> 4095876 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 4095877 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 4095877 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 4095878 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/tempDir-001
[junit4] 2> 4095878 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 4095878 INFO (Thread-8236) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 4095878 INFO (Thread-8236) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 4095880 ERROR (Thread-8236) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4095978 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:47964
[junit4] 2> 4095984 INFO (jetty-launcher-5933-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4095984 INFO (jetty-launcher-5933-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 4095986 INFO (jetty-launcher-5933-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5210b25b{/solr,null,AVAILABLE}
[junit4] 2> 4095987 INFO (jetty-launcher-5933-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4f2b0a6a{HTTP/1.1,[http/1.1]}{127.0.0.1:35045}
[junit4] 2> 4095987 INFO (jetty-launcher-5933-thread-2) [ ] o.e.j.s.Server Started @4098657ms
[junit4] 2> 4095987 INFO (jetty-launcher-5933-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35045}
[junit4] 2> 4095987 ERROR (jetty-launcher-5933-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4095987 INFO (jetty-launcher-5933-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 4095987 INFO (jetty-launcher-5933-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e1a04fe{/solr,null,AVAILABLE}
[junit4] 2> 4095987 INFO (jetty-launcher-5933-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4095987 INFO (jetty-launcher-5933-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4095988 INFO (jetty-launcher-5933-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-11T14:37:50.574Z
[junit4] 2> 4095988 INFO (jetty-launcher-5933-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1e48cc{HTTP/1.1,[http/1.1]}{127.0.0.1:54327}
[junit4] 2> 4095988 INFO (jetty-launcher-5933-thread-1) [ ] o.e.j.s.Server Started @4098657ms
[junit4] 2> 4095988 INFO (jetty-launcher-5933-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54327}
[junit4] 2> 4095988 ERROR (jetty-launcher-5933-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 4095988 INFO (jetty-launcher-5933-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.2.0
[junit4] 2> 4095988 INFO (jetty-launcher-5933-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 4095988 INFO (jetty-launcher-5933-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 4095988 INFO (jetty-launcher-5933-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-10-11T14:37:50.574Z
[junit4] 2> 4095990 INFO (jetty-launcher-5933-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 4095990 INFO (jetty-launcher-5933-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 4095998 INFO (jetty-launcher-5933-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:47964/solr
[junit4] 2> 4095999 INFO (jetty-launcher-5933-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:47964/solr
[junit4] 2> 4096179 INFO (jetty-launcher-5933-thread-1) [n:127.0.0.1:54327_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4096179 INFO (jetty-launcher-5933-thread-2) [n:127.0.0.1:35045_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 4096180 INFO (jetty-launcher-5933-thread-1) [n:127.0.0.1:54327_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54327_solr
[junit4] 2> 4096181 INFO (jetty-launcher-5933-thread-2) [n:127.0.0.1:35045_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35045_solr
[junit4] 2> 4096182 INFO (jetty-launcher-5933-thread-1) [n:127.0.0.1:54327_solr ] o.a.s.c.Overseer Overseer (id=98810768291659782-127.0.0.1:54327_solr-n_0000000000) starting
[junit4] 2> 4096183 INFO (zkCallback-5944-thread-1-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4096184 INFO (zkCallback-5945-thread-1-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 4096190 INFO (jetty-launcher-5933-thread-1) [n:127.0.0.1:54327_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54327_solr
[junit4] 2> 4096192 INFO (zkCallback-5944-thread-1-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4096192 INFO (zkCallback-5945-thread-1-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 4096337 INFO (jetty-launcher-5933-thread-1) [n:127.0.0.1:54327_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54327.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44cb5404
[junit4] 2> 4096342 INFO (jetty-launcher-5933-thread-1) [n:127.0.0.1:54327_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54327.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44cb5404
[junit4] 2> 4096342 INFO (jetty-launcher-5933-thread-1) [n:127.0.0.1:54327_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54327.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44cb5404
[junit4] 2> 4096343 INFO (jetty-launcher-5933-thread-1) [n:127.0.0.1:54327_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/tempDir-001/node1/.
[junit4] 2> 4096415 INFO (jetty-launcher-5933-thread-2) [n:127.0.0.1:35045_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35045.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44cb5404
[junit4] 2> 4096420 INFO (jetty-launcher-5933-thread-2) [n:127.0.0.1:35045_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35045.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44cb5404
[junit4] 2> 4096420 INFO (jetty-launcher-5933-thread-2) [n:127.0.0.1:35045_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35045.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44cb5404
[junit4] 2> 4096421 INFO (jetty-launcher-5933-thread-2) [n:127.0.0.1:35045_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/tempDir-001/node2/.
[junit4] 2> 4096447 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 4096448 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:47964/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 4096474 WARN (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 4096483 WARN (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 4096486 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 4096500 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_serv1.vm3_64164_hdfs____ak4xgy/webapp
[junit4] 2> 4096976 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@serv1-vm3:64164
[junit4] 2> 4097083 WARN (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 4097083 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 4097098 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_44149_datanode____.1saazn/webapp
[junit4] 2> 4097561 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44149
[junit4] 2> 4097622 WARN (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 4097623 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 4097647 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_61371_datanode____.a5ky1z/webapp
[junit4] 2> 4097661 ERROR (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/tempDir-002/hdfsBaseDir/data/data2/]] heartbeating to serv1-vm3/127.0.0.1:33162) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 4097673 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6c99043d39c13: from storage DS-60a7c451-8163-419e-ad88-e01eccf25e5b node DatanodeRegistration(127.0.0.1:45042, datanodeUuid=755af27a-897e-4f71-a77c-210a2c10938b, infoPort=46523, infoSecurePort=0, ipcPort=34821, storageInfo=lv=-56;cid=testClusterID;nsid=1532105618;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 4097673 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6c99043d39c13: from storage DS-d89829ff-2db0-46c6-8ebb-e0792dafbdc3 node DatanodeRegistration(127.0.0.1:45042, datanodeUuid=755af27a-897e-4f71-a77c-210a2c10938b, infoPort=46523, infoSecurePort=0, ipcPort=34821, storageInfo=lv=-56;cid=testClusterID;nsid=1532105618;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 4098248 INFO (SUITE-HdfsRecoveryZkTest-seed#[F2218DA170BE7B3C]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:61371
[junit4] 2> 4098320 ERROR (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/tempDir-002/hdfsBaseDir/data/data4/]] heartbeating to serv1-vm3/127.0.0.1:33162) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 4098325 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6c9906ab5fa0e: from storage DS-8a5d2a2c-f054-45ef-b096-1f0815f83797 node DatanodeRegistration(127.0.0.1:34901, datanodeUuid=3068d645-58be-4f0d-9682-70b6b0b72a09, infoPort=32975, infoSecurePort=0, ipcPort=37868, storageInfo=lv=-56;cid=testClusterID;nsid=1532105618;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 4098325 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6c9906ab5fa0e: from storage DS-885e121a-708b-4f87-beea-7144d0535679 node DatanodeRegistration(127.0.0.1:34901, datanodeUuid=3068d645-58be-4f0d-9682-70b6b0b72a09, infoPort=32975, infoSecurePort=0, ipcPort=37868, storageInfo=lv=-56;cid=testClusterID;nsid=1532105618;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 4098421 INFO (TEST-HdfsRecoveryZkTest.test-seed#[F2218DA170BE7B3C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 4098424 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 4098426 INFO (OverseerThreadFactory-13241-thread-1-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 4098536 INFO (OverseerStateUpdate-98810768291659782-127.0.0.1:54327_solr-n_0000000000) [n:127.0.0.1:54327_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"recoverytest",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"recoverytest_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:35045/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 4098540 INFO (OverseerStateUpdate-98810768291659782-127.0.0.1:54327_solr-n_0000000000) [n:127.0.0.1:54327_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"recoverytest",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"recoverytest_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:54327/solr",
[junit4] 2> "type":"NRT"}
[junit4] 2> 4098746 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4098746 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 4098746 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 4098747 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 4098854 INFO (zkCallback-5945-thread-1-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4098854 INFO (zkCallback-5944-thread-1-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4098854 INFO (zkCallback-5945-thread-2-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4098854 INFO (zkCallback-5944-thread-2-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4099760 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 4099760 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
[junit4] 2> 4099769 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema [recoverytest_shard1_replica_n1] Schema name=minimal
[junit4] 2> 4099769 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema [recoverytest_shard1_replica_n2] Schema name=minimal
[junit4] 2> 4099772 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 4099773 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n2' using configuration from collection recoverytest, trusted=true
[junit4] 2> 4099772 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 4099773 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica_n1' using configuration from collection recoverytest, trusted=true
[junit4] 2> 4099773 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54327.solr.core.recoverytest.shard1.replica_n2' (registry 'solr.core.recoverytest.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44cb5404
[junit4] 2> 4099773 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35045.solr.core.recoverytest.shard1.replica_n1' (registry 'solr.core.recoverytest.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@44cb5404
[junit4] 2> 4099773 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://serv1-vm3:33162/data
[junit4] 2> 4099773 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 4099773 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://serv1-vm3:33162/data
[junit4] 2> 4099773 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4099773 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 4099773 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 4099773 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore [[recoverytest_shard1_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/tempDir-001/node1/recoverytest_shard1_replica_n2], dataDir=[hdfs://serv1-vm3:33162/data/recoverytest/core_node4/data/]
[junit4] 2> 4099773 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore [[recoverytest_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_F2218DA170BE7B3C-001/tempDir-001/node2/recoverytest_shard1_replica_n1], dataDir=[hdfs://serv1-vm3:33162/data/recoverytest/core_node3/data/]
[junit4] 2> 4099774 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://serv1-vm3:33162/data/recoverytest/core_node4/data/snapshot_metadata
[junit4] 2> 4099774 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://serv1-vm3:33162/data/recoverytest/core_node3/data/snapshot_metadata
[junit4] 2> 4099781 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4099781 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 4099781 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4099781 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 4099781 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 4099781 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 4099820 WARN (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 4099869 WARN (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.h.HdfsDirectory The NameNode is in SafeMode - Solr will wait 5 seconds and try again.
[junit4] 2> 4104826 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4104827 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://serv1-vm3:33162/data/recoverytest/core_node3/data
[junit4] 2> 4104842 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://serv1-vm3:33162/data/recoverytest/core_node3/data/index
[junit4] 2> 4104848 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4104848 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 4104848 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 4104901 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4104902 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://serv1-vm3:33162/data/recoverytest/core_node4/data
[junit4] 2> 4104908 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4104928 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://serv1-vm3:33162/data/recoverytest/core_node4/data/index
[junit4] 2> 4104934 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 4104934 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 4104934 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 4104990 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 4105144 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34901 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-885e121a-708b-4f87-beea-7144d0535679:NORMAL:127.0.0.1:34901|RBW], ReplicaUC[[DISK]DS-60a7c451-8163-419e-ad88-e01eccf25e5b:NORMAL:127.0.0.1:45042|RBW]]} size 0
[junit4] 2> 4105145 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45042 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-885e121a-708b-4f87-beea-7144d0535679:NORMAL:127.0.0.1:34901|RBW], ReplicaUC[[DISK]DS-d89829ff-2db0-46c6-8ebb-e0792dafbdc3:NORMAL:127.0.0.1:45042|FINALIZED]]} size 0
[junit4] 2> 4105146 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45042 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-60a7c451-8163-419e-ad88-e01eccf25e5b:NORMAL:127.0.0.1:45042|RBW], ReplicaUC[[DISK]DS-8a5d2a2c-f054-45ef-b096-1f0815f83797:NORMAL:127.0.0.1:34901|RBW]]} size 0
[junit4] 2> 4105146 INFO (Block report processor) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34901 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-60a7c451-8163-419e-ad88-e01eccf25e5b:NORMAL:127.0.0.1:45042|RBW], ReplicaUC[[DISK]DS-8a5d2a2c-f054-45ef-b096-1f0815f83797:NORMAL:127.0.0.1:34901|RBW]]} size 0
[junit4] 2> 4105217 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 4105217 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4105217 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 4105217 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 4105217 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 4105217 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 4105228 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4105228 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4105228 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 4105228 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 4105242 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@169651f6[recoverytest_shard1_replica_n1] main]
[junit4] 2> 4105242 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@2cda8d85[recoverytest_shard1_replica_n2] main]
[junit4] 2> 4105243 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 4105244 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 4105244 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 4105244 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 4105245 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4105245 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 4105246 INFO (searcherExecutor-13247-thread-1-processing-n:127.0.0.1:35045_solr x:recoverytest_shard1_replica_n1 s:shard1 c:recoverytest r:core_node3) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.SolrCore [recoverytest_shard1_replica_n1] Registered new searcher Searcher@169651f6[recoverytest_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4105246 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580972302487519232
[junit4] 2> 4105246 INFO (searcherExecutor-13246-thread-1-processing-n:127.0.0.1:54327_solr x:recoverytest_shard1_replica_n2 s:shard1 c:recoverytest r:core_node4) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SolrCore [recoverytest_shard1_replica_n2] Registered new searcher Searcher@2cda8d85[recoverytest_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 4105246 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1580972302487519232
[junit4] 2> 4105253 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 4105253 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 4105254 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/
[junit4] 2> 4105254 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n2 url=http://127.0.0.1:54327/solr START replicas=[http://127.0.0.1:35045/solr/recoverytest_shard1_replica_n1/] nUpdates=100
[junit4] 2> 4105255 INFO (zkCallback-5945-thread-2-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105255 INFO (zkCallback-5944-thread-1-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105255 INFO (zkCallback-5945-thread-1-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105255 INFO (zkCallback-5944-thread-2-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105257 INFO (qtp1597331456-36458) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.c.S.Request [recoverytest_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 4105560 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica_n2 url=http://127.0.0.1:54327/solr DONE. We have no versions. sync failed.
[junit4] 2> 4105560 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 4105560 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 4105560 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 4105566 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/ shard1
[junit4] 2> 4105670 INFO (zkCallback-5945-thread-2-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105670 INFO (zkCallback-5944-thread-2-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105670 INFO (zkCallback-5944-thread-1-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105670 INFO (zkCallback-5945-thread-1-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105717 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 4105719 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=6974
[junit4] 2> 4105824 INFO (zkCallback-5945-thread-2-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105824 INFO (zkCallback-5945-thread-1-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105824 INFO (zkCallback-5944-thread-1-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4105824 INFO (zkCallback-5944-thread-2-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4106255 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica_n1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=7510
[junit4] 2> 4106258 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 4106359 INFO (zkCallback-5945-thread-2-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4106359 INFO (zkCallback-5945-thread-1-processing-n:127.0.0.1:54327_solr) [n:127.0.0.1:54327_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4106359 INFO (zkCallback-5944-thread-2-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4106359 INFO (zkCallback-5944-thread-1-processing-n:127.0.0.1:35045_solr) [n:127.0.0.1:35045_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 4106441 INFO (OverseerCollectionConfigSetProcessor-98810768291659782-127.0.0.1:54327_solr-n_0000000000) [n:127.0.0.1:54327_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 4107259 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=8837
[junit4] 2> 4107261 INFO (TEST-HdfsRecoveryZkTest.test-seed#[F2218DA170BE7B3C]) [ ] o.a.s.c.RecoveryZkTest Indexing 700 documents
[junit4] 2> 4107268 DEBUG (qtp129688217-36467) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://serv1-vm3:33162/data/recoverytest/core_node4/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 4107274 DEBUG (qtp1597331456-36459) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.HdfsTransactionLog Opening new tlog hdfs tlog{file=hdfs://serv1-vm3:33162/data/recoverytest/core_node3/data/tlog/tlog.0000000000000000000 refcount=1}
[junit4] 2> 4107282 INFO (qtp1597331456-36459) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-0 (1580972304603545600)]} 0 13
[junit4] 2> 4107283 INFO (qtp1597331456-36460) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-0 (1580972304602497024)]} 0 12
[junit4] 2> 4107284 INFO (qtp129688217-36467) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1580972304603545600)]} 0 20
[junit4] 2> 4107284 INFO (qtp129688217-36466) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1580972304602497024)]} 0 21
[junit4] 2> 4107286 INFO (qtp1597331456-36453) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-1 (1580972304625565696)]} 0 0
[junit4] 2> 4107286 INFO (qtp129688217-36468) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1580972304625565696)]} 0 1
[junit4] 2> 4107288 INFO (qtp1597331456-36453) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-1 (1580972304627662848)]} 0 0
[junit4] 2> 4107288 INFO (qtp129688217-36452) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1580972304627662848)]} 0 1
[junit4] 2> 4107289 INFO (qtp1597331456-36458) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[1-0 (-1580972304627662849)]} 0 0
[junit4] 2> 4107289 INFO (qtp129688217-36464) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1580972304627662849)]} 0 2
[junit4] 2> 4107290 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[2-0 (-1580972304629760000)]} 0 1
[junit4] 2> 4107291 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1580972304629760000)]} 0 1
[junit4] 2> 4107291 INFO (qtp1597331456-36461) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-2 (1580972304630808576)]} 0 0
[junit4] 2> 4107291 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1580972304630808576)]} 0 1
[junit4] 2> 4107292 INFO (qtp1597331456-36459) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-2 (1580972304631857152)]} 0 1
[junit4] 2> 4107293 INFO (qtp129688217-36467) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1580972304631857152)]} 0 1
[junit4] 2> 4107293 INFO (qtp1597331456-36460) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-3 (1580972304632905728)]} 0 1
[junit4] 2> 4107293 INFO (qtp129688217-36466) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1580972304632905728)]} 0 1
[junit4] 2> 4107294 INFO (qtp1597331456-36457) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-3 (1580972304633954304)]} 0 0
[junit4] 2> 4107295 INFO (qtp129688217-36468) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1580972304633954304)]} 0 1
[junit4] 2> 4107295 INFO (qtp1597331456-36453) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[1-2 (-1580972304635002880)]} 0 0
[junit4] 2> 4107295 INFO (qtp129688217-36452) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-2 (-1580972304635002880)]} 0 1
[junit4] 2> 4107297 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-4 (1580972304637100032)]} 0 0
[junit4] 2> 4107297 INFO (qtp1597331456-36458) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[2-2 (-1580972304636051456)]} 0 1
[junit4] 2> 4107297 INFO (qtp129688217-36452) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1580972304637100032)]} 0 1
[junit4] 2> 4107297 INFO (qtp129688217-36464) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-2 (-1580972304636051456)]} 0 1
[junit4] 2> 4107299 INFO (qtp1597331456-36459) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-4 (1580972304639197185)]} 0 0
[junit4] 2> 4107299 INFO (qtp1597331456-36461) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-5 (1580972304639197184)]} 0 0
[junit4] 2> 4107299 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1580972304639197184)]} 0 1
[junit4] 2> 4107299 INFO (qtp129688217-36467) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1580972304639197185)]} 0 1
[junit4] 2> 4107301 INFO (qtp1597331456-36457) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-5 (1580972304641294336)]} 0 0
[junit4] 2> 4107301 INFO (qtp1597331456-36460) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[1-5 (-1580972304641294337)]} 0 0
[junit4] 2> 4107301 INFO (qtp129688217-36466) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-5 (-1580972304641294337)]} 0 1
[junit4] 2> 4107301 INFO (qtp129688217-36468) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1580972304641294336)]} 0 1
[junit4] 2> 4107303 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[2-5 (-1580972304643391489)]} 0 0
[junit4] 2> 4107303 INFO (qtp1597331456-36453) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-6 (1580972304643391488)]} 0 1
[junit4] 2> 4107303 INFO (qtp129688217-36452) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-5 (-1580972304643391489)]} 0 1
[junit4] 2> 4107303 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1580972304643391488)]} 0 1
[junit4] 2> 4107305 INFO (qtp1597331456-36458) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-6 (1580972304645488640)]} 0 0
[junit4] 2> 4107305 INFO (qtp1597331456-36459) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-7 (1580972304645488641)]} 0 0
[junit4] 2> 4107305 INFO (qtp129688217-36464) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1580972304645488640)]} 0 1
[junit4] 2> 4107305 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1580972304645488641)]} 0 1
[junit4] 2> 4107307 INFO (qtp1597331456-36461) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-7 (1580972304647585792)]} 0 0
[junit4] 2> 4107307 INFO (qtp129688217-36467) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1580972304647585792)]} 0 1
[junit4] 2> 4107307 INFO (qtp1597331456-36457) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-8 (1580972304647585793)]} 0 1
[junit4] 2> 4107308 INFO (qtp129688217-36466) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1580972304647585793)]} 0 1
[junit4] 2> 4107309 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-9 (1580972304649682945)]} 0 0
[junit4] 2> 4107309 INFO (qtp1597331456-36460) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-8 (1580972304649682944)]} 0 1
[junit4] 2> 4107310 INFO (qtp129688217-36468) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1580972304649682944)]} 0 1
[junit4] 2> 4107310 INFO (qtp129688217-36452) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1580972304649682945)]} 0 2
[junit4] 2> 4107312 INFO (qtp1597331456-36458) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[1-9 (-1580972304651780097)]} 0 1
[junit4] 2> 4107312 INFO (qtp1597331456-36453) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-9 (1580972304651780096)]} 0 1
[junit4] 2> 4107312 INFO (qtp129688217-36464) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-9 (-1580972304651780097)]} 0 2
[junit4] 2> 4107312 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1580972304651780096)]} 0 2
[junit4] 2> 4107314 INFO (qtp1597331456-36459) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[2-9 (-1580972304654925824)]} 0 0
[junit4] 2> 4107314 INFO (qtp1597331456-36461) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-10 (1580972304654925825)]} 0 0
[junit4] 2> 4107314 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-9 (-1580972304654925824)]} 0 1
[junit4] 2> 4107315 INFO (qtp129688217-36467) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1580972304654925825)]} 0 1
[junit4] 2> 4107316 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-11 (1580972304657022977)]} 0 0
[junit4] 2> 4107316 INFO (qtp1597331456-36457) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-10 (1580972304657022976)]} 0 0
[junit4] 2> 4107317 INFO (qtp129688217-36466) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1580972304657022976)]} 0 1
[junit4] 2> 4107317 INFO (qtp129688217-36468) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1580972304657022977)]} 0 1
[junit4] 2> 4107319 INFO (qtp1597331456-36458) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-11 (1580972304659120129)]} 0 0
[junit4] 2> 4107319 INFO (qtp1597331456-36460) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-12 (1580972304659120128)]} 0 0
[junit4] 2> 4107319 INFO (qtp129688217-36452) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-12 (1580972304659120128)]} 0 1
[junit4] 2> 4107319 INFO (qtp129688217-36464) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1580972304659120129)]} 0 1
[junit4] 2> 4107321 INFO (qtp1597331456-36459) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-12 (1580972304661217280)]} 0 0
[junit4] 2> 4107321 INFO (qtp1597331456-36453) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[1-11 (-1580972304662265856)]} 0 0
[junit4] 2> 4107321 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-12 (1580972304661217280)]} 0 1
[junit4] 2> 4107321 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-11 (-1580972304662265856)]} 0 1
[junit4] 2> 4107323 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-13 (1580972304664363008)]} 0 0
[junit4] 2> 4107323 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-13 (1580972304664363008)]} 0 1
[junit4] 2> 4107325 INFO (qtp1597331456-36457) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-14 (1580972304666460160)]} 0 0
[junit4] 2> 4107325 INFO (qtp129688217-36468) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-14 (1580972304666460160)]} 0 1
[junit4] 2> 4107326 INFO (qtp1597331456-36458) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[1-12 (-1580972304667508736)]} 0 0
[junit4] 2> 4107327 INFO (qtp129688217-36452) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-12 (-1580972304667508736)]} 0 1
[junit4] 2> 4107328 INFO (qtp1597331456-36460) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-15 (1580972304669605888)]} 0 0
[junit4] 2> 4107328 INFO (qtp129688217-36464) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-15 (1580972304669605888)]} 0 1
[junit4] 2> 4107330 INFO (qtp1597331456-36459) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-16 (1580972304671703040)]} 0 0
[junit4] 2> 4107330 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-16 (1580972304671703040)]} 0 1
[junit4] 2> 4107332 INFO (qtp1597331456-36453) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-17 (1580972304673800192)]} 0 0
[junit4] 2> 4107332 INFO (qtp129688217-36466) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-17 (1580972304673800192)]} 0 1
[junit4] 2> 4107334 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[1-18 (1580972304675897344)]} 0 0
[junit4] 2> 4107334 INFO (qtp1597331456-36461) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[2-11 (-1580972304663314432)]} 0 12
[junit4] 2> 4107334 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-18 (1580972304675897344)]} 0 1
[junit4] 2> 4107334 INFO (qtp129688217-36467) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-11 (-1580972304663314432)]} 0 13
[junit4] 2> 4107336 INFO (qtp1597331456-36458) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-13 (1580972304677994497)]} 0 0
[junit4] 2> 4107336 INFO (qtp129688217-36452) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-13 (1580972304677994497)]} 0 1
[junit4] 2> 4107338 INFO (qtp1597331456-36460) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-14 (1580972304680091648)]} 0 0
[junit4] 2> 4107338 INFO (qtp129688217-36464) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-14 (1580972304680091648)]} 0 1
[junit4] 2> 4107339 INFO (qtp1597331456-36459) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{delete=[2-12 (-1580972304681140224)]} 0 0
[junit4] 2> 4107340 INFO (qtp129688217-36465) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-12 (-1580972304681140224)]} 0 1
[junit4] 2> 4107341 INFO (qtp1597331456-36453) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-15 (1580972304683237376)]} 0 0
[junit4] 2> 4107342 INFO (qtp129688217-36466) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-15 (1580972304683237376)]} 0 1
[junit4] 2> 4107343 INFO (qtp1597331456-36456) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-16 (1580972304685334528)]} 0 0
[junit4] 2> 4107344 INFO (qtp129688217-36463) [n:127.0.0.1:54327_solr c:recoverytest s:shard1 r:core_node4 x:recoverytest_shard1_replica_n2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n2] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-16 (1580972304685334528)]} 0 1
[junit4] 2> 4107345 INFO (qtp1597331456-36461) [n:127.0.0.1:35045_solr c:recoverytest s:shard1 r:core_node3 x:recoverytest_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54327/solr/recoverytest_shard1_replica_n2/&wt=javabin&version=2}{add=[2-17 (1580972304687431680)]} 0 0
[junit4] 2> 4107345 INFO (qtp129688217-36467) [n:127.0.0.1:54327_solr c:recov
[...truncated too long message...]
TestSolrCoreParser, TestMissingGroups, TestSystemIdResolver, TestZkAclsWithHadoopAuth, DocValuesMissingTest, TestFastLRUCache, TestExclusionRuleCollectionAccess, AutoscalingHistoryHandlerTest, TestLRUCache, TestSha256AuthenticationProvider, TestComponentsName, TestMaxTokenLenTokenizer, TestXmlQParserPlugin, TestLuceneMatchVersion, SpellCheckCollatorWithCollapseTest, RecoveryAfterSoftCommitTest, TestSSLRandomization, IndexSchemaRuntimeFieldTest, TestSubQueryTransformer, BigEndianAscendingWordSerializerTest, ZkNodePropsTest, CurrencyFieldTypeTest, DistributedFacetPivotSmallTest, SparseHLLTest, TestSchemaNameResource, TestQueryTypes, HdfsUnloadDistributedZkTest, TestStressLucene, TestDocumentBuilder, TestStandardQParsers, TestIndexSearcher, TestCloudPivotFacet, TestPartialUpdateDeduplication, V2ApiIntegrationTest, TestPerFieldSimilarity, LeaderFailureAfterFreshStartTest, TestTrackingShardHandlerFactory, PingRequestHandlerTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestBadConfig, CollectionsAPISolrJTest, AutoScalingHandlerTest, TestRandomFaceting, AssignBackwardCompatibilityTest, CopyFieldTest, FileBasedSpellCheckerTest, TestRestoreCore, TestLuceneIndexBackCompat, TestConfigsApi, BJQParserTest, TestAddFieldRealTimeGet, TestConfigSets, TestEmbeddedSolrServerConstructors, TermVectorComponentDistributedTest, UUIDUpdateProcessorFallbackTest, TestPointFields, CacheHeaderTest, TestElisionMultitermQuery, TestSolrJ, JsonLoaderTest, TestMultiValuedNumericRangeQuery, TestCollationField, TestTolerantUpdateProcessorCloud, TestPostingsSolrHighlighter, ResponseHeaderTest, TestFieldCollectionResource, TestSearchPerf, LukeRequestHandlerTest, TestAuthenticationFramework, TestCustomStream, TestCrossCoreJoin, TestCloudSchemaless, TestFilteredDocIdSet, BasicFunctionalityTest, InfixSuggestersTest, OverseerTaskQueueTest, TestFreeTextSuggestions, TestSolr4Spatial, SolrCoreCheckLockOnStartupTest, TestNoOpRegenerator, TestLegacyNumericUtils, ZkSolrClientTest, TestReloadAndDeleteDocs, TestQueryUtils, AnalyticsQueryTest, HdfsChaosMonkeyNothingIsSafeTest, OverseerCollectionConfigSetProcessorTest, PreAnalyzedFieldTest, TestUseDocValuesAsStored2, LeaderElectionContextKeyTest, SolrGraphiteReporterTest, ImplicitSnitchTest, ActionThrottleTest, CloudMLTQParserTest, SolrMetricsIntegrationTest, TestDistributedGrouping, TestLockTree, CollectionTooManyReplicasTest, ExternalFileFieldSortTest, UpdateRequestProcessorFactoryTest, SystemInfoHandlerTest, BufferStoreTest, DistributedQueryComponentCustomSortTest, SuggesterTSTTest, LeaderInitiatedRecoveryOnShardRestartTest, TestRangeQuery, EchoParamsTest, DistributedSuggestComponentTest, CdcrReplicationHandlerTest, SolrCloudExampleTest, TestBinaryResponseWriter, SearchHandlerTest, TestRemoteStreaming, TestFieldCacheSort, TestInPlaceUpdatesStandalone, TestCustomSort, TestShortCircuitedRequests, SolrIndexConfigTest, HdfsLockFactoryTest, TestSolrCoreSnapshots, SolrJmxReporterTest, FileUtilsTest, VMParamsZkACLAndCredentialsProvidersTest, DistribCursorPagingTest, TestNestedDocsSort, TestNumericRangeQuery64, TestHalfAndHalfDocValues, TestDistribIDF, TestBulkSchemaConcurrent, TestStressLiveNodes, SolrPluginUtilsTest, TestLMJelinekMercerSimilarityFactory, TestLegacyField, ShufflingReplicaListTransformerTest, SpellingQueryConverterTest, TermsComponentTest, TestRecoveryHdfs, ChaosMonkeySafeLeaderWithPullReplicasTest, RulesTest, TestCloudInspectUtil, SpellCheckCollatorTest, WrapperMergePolicyFactoryTest, LoggingHandlerTest, TestPullReplica, InfoHandlerTest, TestZkChroot, TestMergePolicyConfig, DistributedVersionInfoTest, TestDistribDocBasedVersion, DocumentAnalysisRequestHandlerTest, SpellPossibilityIteratorTest, TestOrdValues, TestFastOutputStream, RankQueryTest, HdfsBasicDistributedZk2Test, TestRebalanceLeaders, DistributedIntervalFacetingTest, TestRealTimeGet, CdcrUpdateLogTest, TestApiFramework, TestTrie, ConfigSetsAPITest, DOMUtilTest, TestNamedUpdateProcessors, TestSearcherReuse, TestPKIAuthenticationPlugin, TestExportWriter, TestTrieFacet, StatelessScriptUpdateProcessorFactoryTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, LeaderElectionTest, TestRandomDVFaceting, ShardRoutingCustomTest, TestRecovery, TestJoin, TestReload, HardAutoCommitTest, StatsComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestSort, TestFiltering, TestLazyCores, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, SolrRequestParserTest, PolyFieldTest, SchemaVersionSpecificBehaviorTest, TestAtomicUpdateErrorCases, QueryEqualityTest, TestSolrDeletionPolicy1, DebugComponentTest, IndexBasedSpellCheckerTest, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, XmlUpdateRequestHandlerTest, TestValueSourceCache, TestIndexingPerformance, TestSolrQueryParser, IndexSchemaTest, TestJmxIntegration, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, CSVRequestHandlerTest, HighlighterConfigTest, SOLR749Test, UpdateParamsTest, AlternateDirectoryTest, TestSolrIndexConfig, BadComponentTest, MultiTermTest, SampleTest, TestDocSet, NumericFieldsTest, OutputWriterTest, DirectSolrConnectionTest, TestPhraseSuggestions, TestCharFilters, TestCodecSupport, SynonymTokenizerTest, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, ResourceLoaderTest, ScriptEngineTest, OpenExchangeRatesOrgProviderTest, PluginInfoTest, TestSuggestSpellingConverter, DateFieldTest, ClusterStateTest, TestUtils, UUIDFieldTest, DistributedMLTComponentTest, TestRTGBase, CursorPagingTest, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestSimpleTrackingShardHandler, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CleanupOldIndexTest, CreateCollectionCleanupTest, CustomCollectionTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteReplicaTest, DistribJoinFromCollectionTest, HttpPartitionTest, LeaderInitiatedRecoveryOnCommitTest, MoveReplicaHDFSTest, NodeMutatorTest, OverseerRolesTest, RemoteQueryErrorTest, ReplaceNodeTest, ReplicationFactorTest, RestartWhileUpdatingTest, RollingRestartTest, SSLMigrationTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, TestCloudDeleteByQuery, TestCloudJSONFacetJoinDomain, TestCloudPseudoReturnFields, TestCloudRecovery, TestCloudSearcherWarming, TestClusterProperties, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestDownShardTolerantSearch, TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, TestLeaderInitiatedRecoveryThread, TestLocalFSCloudBackupRestore, TestOnReconnectListenerSupport, TestPrepRecovery, TestRandomFlRTGCloud, TestReplicaProperties, TestRequestForwarding, TestSegmentSorting, TestSkipOverseerOperations, TestStressCloudBlindAtomicUpdates, TestStressInPlaceUpdates, AutoAddReplicasIntegrationTest, HttpTriggerListenerTest, NodeLostTriggerTest, TestPolicyCloud, HdfsBasicDistributedZkTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoverLeaseTest, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=F2218DA170BE7B3C -Dtests.slow=true -Dtests.locale=fr-CA -Dtests.timezone=Pacific/Fiji -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J1 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:132)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:161)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:116)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:108)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:774)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:836)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1089)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:959)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:844)
[junit4] > at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1029)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:642)
[junit4] > at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([F2218DA170BE7B3C]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:294)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] Completed [569/744 (1!)] on J1 in 43.05s, 1 test, 1 failure <<< FAILURES!
[...truncated 47524 lines...]